General.OutsideOfAnyTestCase: 03:03:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28 in 

03:03:21.217 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:03:21.217 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasksldxbt/test-tmp
03:03:21.217 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
03:03:21.217 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasksldxbt/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
03:03:21.218 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891"
  extra_master_flags: "--txn_manager_enabled"
  extra_master_flags: "--raft_heartbeat_interval_ms=100"
  extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
  extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
  extra_tserver_flags: "--enable_txn_system_client_init=true"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

03:03:21.544 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
03:03:21.545 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

03:03:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.547259 17185 mini_oidc.cc:152] Starting JWKS server
03:03:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.550019 17185 webserver.cc:460] Webserver started at http://127.0.0.1:45407/ using document root <none> and password file <none>
03:03:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.550221 17185 mini_oidc.cc:189] Starting OIDC Discovery server
03:03:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.550807 17185 webserver.cc:460] Webserver started at http://127.0.0.1:43245/ using document root <none> and password file <none>
03:03:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T03:03:21Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:03:21.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T03:03:21Z Disabled control of system clock
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.581426 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/logs
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data/info.pb
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.126:36191
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.126
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:03:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.902133 17196 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.902436 17196 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.902520 17196 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.909809 17196 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.909930 17196 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.909970 17196 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.910018 17196 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.910055 17196 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.910108 17196 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.910140 17196 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45407/jwks.json
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.919488 17196 master_runner.cc:386] Master server non-default flags:
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.126:36191
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data/info.pb
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.126
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/logs
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.920847 17196 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.922540 17196 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.934266 17201 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.934187 17204 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:21.934192 17202 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.935258 17196 server_base.cc:1034] running on GCE node
03:03:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.935987 17196 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.937388 17196 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.938584 17196 hybrid_clock.cc:648] HybridClock initialized: now 1781924601938541 us; error 36 us; skew 500 ppm
03:03:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.938889 17196 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.941151 17196 webserver.cc:460] Webserver started at http://127.16.200.126:46681/ using document root <none> and password file <none>
03:03:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.941743 17196 fs_manager.cc:359] Metadata directory not provided
03:03:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.941835 17196 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.942081 17196 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.944620 17196 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data/instance:
03:03:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d693881715e435abf6ea251c71c8288"
03:03:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:21 on dist-test-slave-214v"
03:03:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.945323 17196 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal/instance:
03:03:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d693881715e435abf6ea251c71c8288"
03:03:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:21 on dist-test-slave-214v"
03:03:21.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.950387 17196 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
03:03:21.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.954082 17210 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.955607 17196 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
03:03:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.955771 17196 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal
03:03:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d693881715e435abf6ea251c71c8288"
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:21 on dist-test-slave-214v"
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.955971 17196 fs_report.cc:383] FS layout report
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data/data
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:21.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.971192 17196 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.972023 17196 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.972285 17196 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.994560 17196 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.126:36191
03:03:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.994607 17271 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.126:36191 every 8 connection(s)
03:03:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.996264 17196 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/data/info.pb
03:03:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.998037 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17196
03:03:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:21.998250 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-0/wal/instance
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.000321 17272 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.000326 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/logs
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.125:33579
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.125
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:03:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.007181 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.023977 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.025095 17212 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.026646 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.076066 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } attempt: 1
03:03:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.078163 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.146481 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } attempt: 2
03:03:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.148559 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.239861 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } attempt: 3
03:03:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.241909 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.332980 17274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.333281 17274 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.333376 17274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.340833 17274 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.340946 17274 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.340982 17274 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.341029 17274 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.341065 17274 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.341120 17274 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.341151 17274 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45407/jwks.json
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.350469 17274 master_runner.cc:386] Master server non-default flags:
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.125:33579
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.125
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/logs
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.351764 17274 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:22.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.353557 17274 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.365715 17282 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.365782 17285 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.365832 17283 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.366935 17274 server_base.cc:1034] running on GCE node
03:03:22.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.367817 17274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.369422 17274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:22.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.370687 17274 hybrid_clock.cc:648] HybridClock initialized: now 1781924602370616 us; error 57 us; skew 500 ppm
03:03:22.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.371104 17274 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.376832 17274 webserver.cc:460] Webserver started at http://127.16.200.125:39307/ using document root <none> and password file <none>
03:03:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.377616 17274 fs_manager.cc:359] Metadata directory not provided
03:03:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.377754 17274 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:22.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.378098 17274 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:22.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.381047 17274 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/instance:
03:03:22.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:22.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.381935 17274 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal/instance:
03:03:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.385207 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } attempt: 4
03:03:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.387152 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.387742 17274 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
03:03:22.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.391294 17291 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:22.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.392767 17274 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.392907 17274 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.393088 17274 fs_report.cc:383] FS layout report
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/data
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.412277 17274 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.413233 17274 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.413519 17274 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.435680 17274 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.125:33579
03:03:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.435719 17352 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.125:33579 every 8 connection(s)
03:03:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.437163 17274 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.440979 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17274
03:03:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.441160 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal/instance
03:03:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.441670 17353 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.442468 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/logs
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.124:40167
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.124
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:03:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.449752 17353 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.465835 17353 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.475641 17353 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.476938 17294 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.478871 17353 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.528237 17353 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 1
03:03:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.530685 17353 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.598979 17353 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 2
03:03:22.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.601207 17353 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.686443 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } attempt: 5
03:03:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.692467 17353 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 3
03:03:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.694340 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.694586 17353 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.696311 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.747617 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 1
03:03:22.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.749975 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.772603 17355 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.772910 17355 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.772998 17355 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780275 17355 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780382 17355 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780478 17355 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780544 17355 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780583 17355 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780637 17355 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.780670 17355 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45407/jwks.json
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.790045 17355 master_runner.cc:386] Master server non-default flags:
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.124:40167
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.124
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/logs
03:03:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.791286 17355 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.793123 17355 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.805526 17363 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.805585 17366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.805609 17364 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.806757 17355 server_base.cc:1034] running on GCE node
03:03:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.807492 17355 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:22.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.809088 17355 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:22.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.810382 17355 hybrid_clock.cc:648] HybridClock initialized: now 1781924602810329 us; error 41 us; skew 500 ppm
03:03:22.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.810774 17355 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.813207 17355 webserver.cc:460] Webserver started at http://127.16.200.124:35103/ using document root <none> and password file <none>
03:03:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.813898 17355 fs_manager.cc:359] Metadata directory not provided
03:03:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.814042 17355 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.814359 17355 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.817147 17355 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/instance:
03:03:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c105f56a90947ff95a026f305fc5904"
03:03:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.817989 17355 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal/instance:
03:03:22.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c105f56a90947ff95a026f305fc5904"
03:03:22.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.818320 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 2
03:03:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.820452 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.823660 17355 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
03:03:22.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.827270 17372 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.828724 17355 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.828935 17355 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c105f56a90947ff95a026f305fc5904"
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.829165 17355 fs_report.cc:383] FS layout report
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/data
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.837836 17353 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 4
03:03:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.839715 17353 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.847422 17355 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.848325 17355 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.848676 17355 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:22.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.870436 17355 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.124:40167
03:03:22.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.870461 17433 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.124:40167 every 8 connection(s)
03:03:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.871915 17355 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:22.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.875725 17434 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.880766 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17355
03:03:22.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.880935 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal/instance
03:03:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.882285 17434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.898458 17434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.907460 17434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
03:03:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.919107 17434 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Bootstrap starting.
03:03:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.922405 17434 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Neither blocks nor log segments found. Creating new log.
03:03:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.923470 17434 log.cc:824] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Log is configured to *not* fsync() on all Append() calls
03:03:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.926165 17434 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: No bootstrap required, opened a new log
03:03:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.926683 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 3
03:03:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.937318 17434 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.937736 17434 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.937839 17434 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c105f56a90947ff95a026f305fc5904, State: Initialized, Role: FOLLOWER
03:03:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.938493 17434 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.939558 17272 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: Bootstrap starting.
03:03:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.939407 17443 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.939790 17443 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: FOLLOWER
03:03:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.940739 17434 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: configured and running, proceeding with master startup.
03:03:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.943984 17272 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: Neither blocks nor log segments found. Creating new log.
03:03:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.945622 17272 log.cc:824] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: Log is configured to *not* fsync() on all Append() calls
03:03:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.951792 17454 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:03:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.952047 17454 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.954939 17272 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: No bootstrap required, opened a new log
03:03:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.949573 17355 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: OK, instance_id { permanent_uuid: "2d693881715e435abf6ea251c71c8288" instance_seqno: 1781924601989123 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:03:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.959296 17355 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.964006 17443 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.964325 17443 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.965919 17443 leader_election.cc:288] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579)
03:03:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.966490 17272 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.966902 17272 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.967036 17272 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d693881715e435abf6ea251c71c8288, State: Initialized, Role: FOLLOWER
03:03:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.967727 17272 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.973325 17457 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.974032 17457 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: FOLLOWER
03:03:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.974648 17272 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: configured and running, proceeding with master startup.
03:03:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.974748 17237 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288" is_pre_election: true
03:03:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:22.975021 17216 tablet.cc:2369] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.975355 17237 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1c105f56a90947ff95a026f305fc5904 in term 0.
03:03:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.976046 17376 leader_election.cc:302] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288; no voters: 
03:03:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.976519 17443 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:03:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.976675 17443 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.976788 17443 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 0 FOLLOWER]: Advancing to term 1
03:03:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.979949 17443 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.981276 17237 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288"
03:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.981513 17237 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 0 FOLLOWER]: Advancing to term 1
03:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.984735 17237 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1c105f56a90947ff95a026f305fc5904 in term 1.
03:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.985258 17376 leader_election.cc:302] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288; no voters: 
03:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.985764 17443 leader_election.cc:288] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 1 election: Requested vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579)
03:03:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.986014 17443 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Leader election won for term 1
03:03:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.986172 17443 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 LEADER]: Becoming Leader. State: Replica: 1c105f56a90947ff95a026f305fc5904, State: Running, Role: LEADER
03:03:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.986891 17443 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:22.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.991991 17460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1c105f56a90947ff95a026f305fc5904. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:22.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:22.992950 17460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: LEADER
03:03:23.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.005632 17464 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:03:23.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.013808 17464 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:03:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.030597 17237 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Refusing update from remote peer 1c105f56a90947ff95a026f305fc5904: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:23.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.031582 17460 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.039382 17457 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1c105f56a90947ff95a026f305fc5904. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.039707 17457 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: FOLLOWER
03:03:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.049717 17460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.050028 17460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: LEADER
03:03:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.056923 17464 catalog_manager.cc:1347] Generated new cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.058014 17464 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:03:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.064209 17473 mvcc.cc:205] Tried to move back new op lower bound from 7298763173993992192 to 7298763173841915904. Current Snapshot: MvccSnapshot[applied={T|T < 7298763173993992192}]
03:03:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.045281 17196 master_runner.cc:186] Error getting master registration for 127.16.200.125:33579: OK, instance_id { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" instance_seqno: 1781924602430245 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:03:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.071359 17457 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.073556 17457 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: FOLLOWER
03:03:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.074019 17196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.077337 17472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:03:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.077473 17472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:03:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.138895 17353 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } attempt: 5
03:03:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.145464 17464 catalog_manager.cc:1370] Generated new certificate authority record
03:03:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.149021 17464 catalog_manager.cc:1504] Loading token signing keys...
03:03:23.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.156821 17353 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Bootstrap starting.
03:03:23.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.163120 17353 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Neither blocks nor log segments found. Creating new log.
03:03:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.166085 17353 log.cc:824] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Log is configured to *not* fsync() on all Append() calls
03:03:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.171039 17464 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Generated new TSK 0
03:03:23.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.173051 17464 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:03:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.191638 17353 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: No bootstrap required, opened a new log
03:03:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.200444 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal
03:03:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/logs
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data/info.pb
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.65:0
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.65
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.65
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:03:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.207567 17353 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.208026 17353 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.208132 17353 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0cafa7f6fbec485b99c2ecdd6c8af1eb, State: Initialized, Role: FOLLOWER
03:03:23.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.209041 17353 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.216181 17480 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.216727 17480 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.218881 17353 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: configured and running, proceeding with master startup.
03:03:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.218956 17318 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" is_pre_election: true
03:03:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.219583 17318 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1c105f56a90947ff95a026f305fc5904 in term 0.
03:03:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.220362 17316 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 0 FOLLOWER]: Advancing to term 1
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.218904 17317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.228227 17317 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46344 (request call id 2) took 242 ms (client timeout 307 ms). Trace:
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.228446 17317 rpcz_store.cc:269] 0620 03:03:22.985649 (+     0us) service_pool.cc:170] Inserting onto call queue
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:22.985737 (+    88us) service_pool.cc:227] Handling call
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:23.228158 (+242421us) inbound_call.cc:173] Queueing success response
03:03:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:03:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.232146 17316 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Refusing update from remote peer 1c105f56a90947ff95a026f305fc5904: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.233466 17443 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.294565 17487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.294588 17480 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1c105f56a90947ff95a026f305fc5904. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.295307 17487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.295370 17480 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:23.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.329239 17496 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:03:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.335188 17443 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.335692 17443 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: LEADER
03:03:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.671187 17479 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.671536 17479 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.671685 17479 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:23.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.679666 17479 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.679889 17479 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:03:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.679955 17479 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:03:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.680024 17479 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:03:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.680142 17479 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.200.65
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.690217 17479 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.65:0
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data/info.pb
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.65
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.65
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/logs
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:23.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.691927 17479 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:23.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.693683 17479 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.706903 17502 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.706924 17505 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:23.707175 17503 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.708176 17479 server_base.cc:1034] running on GCE node
03:03:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.708940 17479 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.709929 17479 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.711127 17479 hybrid_clock.cc:648] HybridClock initialized: now 1781924603711058 us; error 66 us; skew 500 ppm
03:03:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.711408 17479 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:23.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.713891 17479 webserver.cc:460] Webserver started at http://127.16.200.65:33751/ using document root <none> and password file <none>
03:03:23.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.714608 17479 fs_manager.cc:359] Metadata directory not provided
03:03:23.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.714697 17479 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:23.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.714965 17479 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.717703 17479 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data/instance:
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1910a86c11064db0b0a19531b509e6ac"
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:23 on dist-test-slave-214v"
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.718441 17479 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal/instance:
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1910a86c11064db0b0a19531b509e6ac"
03:03:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:23 on dist-test-slave-214v"
03:03:23.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.724123 17479 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
03:03:23.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.728464 17511 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.730180 17479 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.730351 17479 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1910a86c11064db0b0a19531b509e6ac"
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:23 on dist-test-slave-214v"
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.730551 17479 fs_report.cc:383] FS layout report
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal
03:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data/data
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.749984 17479 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.750813 17479 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.751066 17479 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.754158 17479 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.754288 17479 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
03:03:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.754418 17479 ts_tablet_manager.cc:610] Registered 0 tablets
03:03:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.754487 17479 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
03:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.819485 17479 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.65:36499
03:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.819521 17635 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.65:36499 every 8 connection(s)
03:03:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.820920 17479 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/data/info.pb
03:03:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.824699 17637 heartbeater.cc:347] Connected to a master server at 127.16.200.126:36191
03:03:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.825034 17637 heartbeater.cc:464] Registering TS with master...
03:03:23.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.825887 17637 heartbeater.cc:511] Master 127.16.200.126:36191 requested a full tablet report, sending...
03:03:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.828377 17227 ts_manager.cc:194] Registered new tserver with Master: 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499)
03:03:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.829960 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17479
03:03:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.830199 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-0/wal/instance
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.834580 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/logs
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data/info.pb
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.66:0
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.66
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.66
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.835304 17638 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.835456 17638 heartbeater.cc:464] Registering TS with master...
03:03:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.835986 17638 heartbeater.cc:511] Master 127.16.200.125:33579 requested a full tablet report, sending...
03:03:23.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.837709 17308 ts_manager.cc:194] Registered new tserver with Master: 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499)
03:03:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.838721 17636 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.838835 17636 heartbeater.cc:464] Registering TS with master...
03:03:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.839159 17636 heartbeater.cc:511] Master 127.16.200.124:40167 requested a full tablet report, sending...
03:03:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.841007 17389 ts_manager.cc:194] Registered new tserver with Master: 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499)
03:03:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:23.842480 17389 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.200.65:42053
03:03:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.079515 17472 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.079610 17472 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: loading cluster ID for follower catalog manager: success
03:03:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.082237 17472 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: acquiring CA information for follower catalog manager: success
03:03:24.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.084012 17472 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.188922 17641 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.189242 17641 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.189401 17641 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.196823 17641 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.197009 17641 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:03:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.197068 17641 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:03:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.197134 17641 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:03:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.197250 17641 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.200.66
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.210048 17641 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.66:0
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data/info.pb
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.66
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.66
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/logs
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.211822 17641 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.213721 17641 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.227919 17646 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.227979 17649 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.227895 17647 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.229225 17641 server_base.cc:1034] running on GCE node
03:03:24.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.229966 17641 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.230965 17641 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:24.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.232280 17641 hybrid_clock.cc:648] HybridClock initialized: now 1781924604232164 us; error 58 us; skew 500 ppm
03:03:24.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.232698 17641 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.235267 17641 webserver.cc:460] Webserver started at http://127.16.200.66:42201/ using document root <none> and password file <none>
03:03:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.236027 17641 fs_manager.cc:359] Metadata directory not provided
03:03:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.236157 17641 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.236557 17641 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.239140 17641 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data/instance:
03:03:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c46998665694072ba111cab69a45c83"
03:03:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.239936 17641 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal/instance:
03:03:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c46998665694072ba111cab69a45c83"
03:03:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.245465 17641 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.008s	sys 0.000s
03:03:24.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.248965 17655 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.250337 17641 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
03:03:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.250548 17641 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal
03:03:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c46998665694072ba111cab69a45c83"
03:03:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.250798 17641 fs_report.cc:383] FS layout report
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data/data
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.275588 17641 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.276695 17641 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.277112 17641 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.281020 17641 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.281147 17641 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
03:03:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.281291 17641 ts_tablet_manager.cc:610] Registered 0 tablets
03:03:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.281350 17641 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
03:03:24.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.336777 17496 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:24.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.336966 17496 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: loading cluster ID for follower catalog manager: success
03:03:24.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.341804 17496 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: acquiring CA information for follower catalog manager: success
03:03:24.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.345376 17496 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.357761 17641 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.66:38815
03:03:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.357860 17777 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.66:38815 every 8 connection(s)
03:03:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.359054 17641 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/data/info.pb
03:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.362052 17779 heartbeater.cc:347] Connected to a master server at 127.16.200.126:36191
03:03:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.362306 17779 heartbeater.cc:464] Registering TS with master...
03:03:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.363193 17779 heartbeater.cc:511] Master 127.16.200.126:36191 requested a full tablet report, sending...
03:03:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.365056 17227 ts_manager.cc:194] Registered new tserver with Master: 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.365694 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17641
03:03:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.365842 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-1/wal/instance
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.372051 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/logs
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data/info.pb
03:03:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.67:0
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.67
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.67
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:03:24.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.378185 17778 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:24.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.378609 17778 heartbeater.cc:464] Registering TS with master...
03:03:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.379258 17778 heartbeater.cc:511] Master 127.16.200.124:40167 requested a full tablet report, sending...
03:03:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.380563 17389 ts_manager.cc:194] Registered new tserver with Master: 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.381735 17389 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.200.66:50007
03:03:24.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.388028 17780 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:24.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.388500 17780 heartbeater.cc:464] Registering TS with master...
03:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.388914 17780 heartbeater.cc:511] Master 127.16.200.125:33579 requested a full tablet report, sending...
03:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.390182 17308 ts_manager.cc:194] Registered new tserver with Master: 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.797013 17783 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.797353 17783 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.797490 17783 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.805281 17783 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.805488 17783 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.805549 17783 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.805619 17783 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.805742 17783 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.200.67
03:03:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.815647 17783 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.67:0
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data/info.pb
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.67
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.200.67
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/logs
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:24.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.817324 17783 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.819697 17783 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.836117 17791 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.836199 17788 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:24.836221 17789 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.837339 17783 server_base.cc:1034] running on GCE node
03:03:24.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.837929 17783 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.838922 17783 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.840162 17783 hybrid_clock.cc:648] HybridClock initialized: now 1781924604840103 us; error 42 us; skew 500 ppm
03:03:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.840608 17783 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.843003 17783 webserver.cc:460] Webserver started at http://127.16.200.67:33095/ using document root <none> and password file <none>
03:03:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.843789 17783 fs_manager.cc:359] Metadata directory not provided
03:03:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.843921 17783 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.844242 17783 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:03:24.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.845872 17636 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.847270 17783 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data/instance:
03:03:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e92d615261964bcf99f19c568f42c5cd"
03:03:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.848193 17783 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal/instance:
03:03:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e92d615261964bcf99f19c568f42c5cd"
03:03:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.853304 17783 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.000s
03:03:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.857019 17797 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.858476 17783 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
03:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.858644 17783 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal
03:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e92d615261964bcf99f19c568f42c5cd"
03:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:24 on dist-test-slave-214v"
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.858867 17783 fs_report.cc:383] FS layout report
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data/data
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.890247 17783 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.891129 17783 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.891446 17783 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:24.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.894688 17783 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.894948 17783 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
03:03:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.895114 17783 ts_tablet_manager.cc:610] Registered 0 tablets
03:03:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.895222 17783 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
03:03:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.972091 17783 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.67:39839
03:03:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.972134 17921 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.67:39839 every 8 connection(s)
03:03:24.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.973399 17783 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/data/info.pb
03:03:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.975958 17923 heartbeater.cc:347] Connected to a master server at 127.16.200.126:36191
03:03:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.976275 17923 heartbeater.cc:464] Registering TS with master...
03:03:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.977303 17923 heartbeater.cc:511] Master 127.16.200.126:36191 requested a full tablet report, sending...
03:03:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.979538 17227 ts_manager.cc:194] Registered new tserver with Master: e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.981046 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 17783
03:03:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.981240 17185 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/ts-2/wal/instance
03:03:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.984906 17924 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.985050 17924 heartbeater.cc:464] Registering TS with master...
03:03:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.985505 17924 heartbeater.cc:511] Master 127.16.200.125:33579 requested a full tablet report, sending...
03:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.985867 17922 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.985994 17922 heartbeater.cc:464] Registering TS with master...
03:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.986428 17922 heartbeater.cc:511] Master 127.16.200.124:40167 requested a full tablet report, sending...
03:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.986722 17308 ts_manager.cc:194] Registered new tserver with Master: e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.987241 17389 ts_manager.cc:194] Registered new tserver with Master: e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:24.988185 17389 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.200.67:55633
03:03:25.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.003201 17185 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:03:25.005 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
03:03:25.006 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

03:03:25.007 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.200.126"
      port: 36191
    }
    bound_http_address {
      host: "127.16.200.126"
      port: 46681
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.200.125"
      port: 33579
    }
    bound_http_address {
      host: "127.16.200.125"
      port: 39307
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.200.124"
      port: 40167
    }
    bound_http_address {
      host: "127.16.200.124"
      port: 35103
    }
  }
}

03:03:25.007 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

03:03:25.007 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.200.65"
      port: 36499
    }
    bound_http_address {
      host: "127.16.200.65"
      port: 33751
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.200.66"
      port: 38815
    }
    bound_http_address {
      host: "127.16.200.66"
      port: 42201
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.200.67"
      port: 39839
    }
    bound_http_address {
      host: "127.16.200.67"
      port: 33095
    }
  }
}

03:03:25.008 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:03:25.009 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
03:03:25.011 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:25.012 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:25.013 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:25.015 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.200.124:40167(127.16.200.124:40167)] Successfully connected to peer
03:03:25.015 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.200.126:36191(127.16.200.126:36191)] Successfully connected to peer
03:03:25.016 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.200.125:33579(127.16.200.125:33579)] Successfully connected to peer
03:03:25.017 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.200.126
03:03:25.017 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.200.124
03:03:25.020 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.200.125
03:03:25.031 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb3cb552d, L:/127.0.0.1:40586 - R:/127.16.200.126:36191] using SASL/PLAIN
03:03:25.041 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0d381c21, L:/127.0.0.1:55684 - R:/127.16.200.125:33579] using SASL/PLAIN
03:03:25.041 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa3a712db, L:/127.0.0.1:56122 - R:/127.16.200.124:40167] using SASL/PLAIN
03:03:25.044 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:03:25.044 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=33), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [32ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, deferred=null) after lookup
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.046176 17389 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56122:
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   hash_schema {
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     num_buckets: 2
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     seed: 0
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.048839 17389 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:03:25.063 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ecbd5643117542d893822cc55e271228
03:03:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.091645 17566 tablet_service.cc:1467] Processing CreateTablet for tablet 37c86a9a5fe2437a9cadff003eddc10d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:03:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.091671 17711 tablet_service.cc:1467] Processing CreateTablet for tablet 37c86a9a5fe2437a9cadff003eddc10d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:03:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.093204 17566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37c86a9a5fe2437a9cadff003eddc10d. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.093220 17711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37c86a9a5fe2437a9cadff003eddc10d. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.091650 17712 tablet_service.cc:1467] Processing CreateTablet for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:03:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.096686 17565 tablet_service.cc:1467] Processing CreateTablet for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:03:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.097218 17565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0cfb25a239cb45979a3dc2d7cb7f5422. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.097229 17712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0cfb25a239cb45979a3dc2d7cb7f5422. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.103443 17848 tablet_service.cc:1467] Processing CreateTablet for tablet 37c86a9a5fe2437a9cadff003eddc10d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:03:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.105127 17848 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37c86a9a5fe2437a9cadff003eddc10d. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.108898 17841 tablet_service.cc:1467] Processing CreateTablet for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:03:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.109439 17841 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0cfb25a239cb45979a3dc2d7cb7f5422. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.114496 17949 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: Bootstrap starting.
03:03:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.117121 17948 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Bootstrap starting.
03:03:25.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.118883 17949 tablet_bootstrap.cc:654] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: Neither blocks nor log segments found. Creating new log.
03:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.120486 17949 log.cc:824] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: Log is configured to *not* fsync() on all Append() calls
03:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.121420 17948 tablet_bootstrap.cc:654] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Neither blocks nor log segments found. Creating new log.
03:03:25.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.122733 17948 log.cc:824] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Log is configured to *not* fsync() on all Append() calls
03:03:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.123188 17951 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Bootstrap starting.
03:03:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.125043 17949 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: No bootstrap required, opened a new log
03:03:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.125319 17949 ts_tablet_manager.cc:1397] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: Time spent bootstrapping tablet: real 0.013s	user 0.005s	sys 0.004s
03:03:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.125386 17948 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: No bootstrap required, opened a new log
03:03:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.125603 17948 ts_tablet_manager.cc:1397] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Time spent bootstrapping tablet: real 0.009s	user 0.000s	sys 0.007s
03:03:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.127346 17951 tablet_bootstrap.cc:654] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Neither blocks nor log segments found. Creating new log.
03:03:25.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.128250 17951 log.cc:824] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Log is configured to *not* fsync() on all Append() calls
03:03:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.130434 17951 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: No bootstrap required, opened a new log
03:03:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.130694 17951 ts_tablet_manager.cc:1397] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Time spent bootstrapping tablet: real 0.008s	user 0.005s	sys 0.000s
03:03:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.136358 17949 raft_consensus.cc:348] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.136538 17948 raft_consensus.cc:348] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.136909 17949 raft_consensus.cc:374] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.136909 17948 raft_consensus.cc:374] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.137002 17948 raft_consensus.cc:729] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1910a86c11064db0b0a19531b509e6ac, State: Initialized, Role: FOLLOWER
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.137005 17949 raft_consensus.cc:729] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c46998665694072ba111cab69a45c83, State: Initialized, Role: FOLLOWER
03:03:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.137669 17948 consensus_queue.cc:260] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.137669 17949 consensus_queue.cc:260] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.139142 17951 raft_consensus.cc:348] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.139505 17951 raft_consensus.cc:374] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.139633 17951 raft_consensus.cc:729] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e92d615261964bcf99f19c568f42c5cd, State: Initialized, Role: FOLLOWER
03:03:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.140242 17951 consensus_queue.cc:260] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.142156 17951 ts_tablet_manager.cc:1428] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Time spent starting tablet: real 0.011s	user 0.007s	sys 0.004s
03:03:25.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.143010 17922 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.146117 17951 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd: Bootstrap starting.
03:03:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.147645 17951 tablet_bootstrap.cc:654] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd: Neither blocks nor log segments found. Creating new log.
03:03:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.149971 17951 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd: No bootstrap required, opened a new log
03:03:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.150111 17951 ts_tablet_manager.cc:1397] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd: Time spent bootstrapping tablet: real 0.006s	user 0.003s	sys 0.000s
03:03:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.150882 17951 raft_consensus.cc:348] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.151101 17951 raft_consensus.cc:374] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.151202 17951 raft_consensus.cc:729] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e92d615261964bcf99f19c568f42c5cd, State: Initialized, Role: FOLLOWER
03:03:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.152333 17949 ts_tablet_manager.cc:1428] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83: Time spent starting tablet: real 0.027s	user 0.015s	sys 0.004s
03:03:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.151454 17951 consensus_queue.cc:260] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.153000 17948 ts_tablet_manager.cc:1428] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Time spent starting tablet: real 0.027s	user 0.021s	sys 0.005s
03:03:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.153322 17778 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.156220 17948 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac: Bootstrap starting.
03:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.158612 17951 ts_tablet_manager.cc:1428] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd: Time spent starting tablet: real 0.008s	user 0.002s	sys 0.000s
03:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.159235 17948 tablet_bootstrap.cc:654] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac: Neither blocks nor log segments found. Creating new log.
03:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.159745 17949 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: Bootstrap starting.
03:03:25.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.163156 17949 tablet_bootstrap.cc:654] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: Neither blocks nor log segments found. Creating new log.
03:03:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.163947 17948 tablet_bootstrap.cc:492] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac: No bootstrap required, opened a new log
03:03:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.164117 17948 ts_tablet_manager.cc:1397] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac: Time spent bootstrapping tablet: real 0.008s	user 0.003s	sys 0.000s
03:03:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.164916 17948 raft_consensus.cc:348] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.165189 17948 raft_consensus.cc:374] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.165284 17948 raft_consensus.cc:729] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1910a86c11064db0b0a19531b509e6ac, State: Initialized, Role: FOLLOWER
03:03:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.165535 17948 consensus_queue.cc:260] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.165997 17949 tablet_bootstrap.cc:492] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: No bootstrap required, opened a new log
03:03:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.166074 17948 ts_tablet_manager.cc:1428] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.000s
03:03:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.166294 17949 ts_tablet_manager.cc:1397] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: Time spent bootstrapping tablet: real 0.007s	user 0.000s	sys 0.005s
03:03:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.168459 17949 raft_consensus.cc:348] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.168701 17949 raft_consensus.cc:374] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.168807 17949 raft_consensus.cc:729] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c46998665694072ba111cab69a45c83, State: Initialized, Role: FOLLOWER
03:03:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.169041 17949 consensus_queue.cc:260] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.169633 17949 ts_tablet_manager.cc:1428] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: Time spent starting tablet: real 0.003s	user 0.001s	sys 0.000s
03:03:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.225950 17925 tablet.cc:2369] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.233748 17956 raft_consensus.cc:482] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.234014 17956 raft_consensus.cc:504] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.235745 17956 leader_election.cc:288] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499), 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.242784 17591 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37c86a9a5fe2437a9cadff003eddc10d" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1910a86c11064db0b0a19531b509e6ac" is_pre_election: true
03:03:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.242769 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37c86a9a5fe2437a9cadff003eddc10d" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83" is_pre_election: true
03:03:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.243356 17591 raft_consensus.cc:2454] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 0.
03:03:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.243356 17732 raft_consensus.cc:2454] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 0.
03:03:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.243978 17801 leader_election.cc:302] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, e92d615261964bcf99f19c568f42c5cd; no voters: 
03:03:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.244354 17956 raft_consensus.cc:2781] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Leader pre-election won for term 1
03:03:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.244524 17956 raft_consensus.cc:482] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.244632 17956 raft_consensus.cc:3037] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Advancing to term 1
03:03:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.246603 17956 raft_consensus.cc:504] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.247212 17956 leader_election.cc:288] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 election: Requested vote from peers 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499), 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.247712 17591 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37c86a9a5fe2437a9cadff003eddc10d" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1910a86c11064db0b0a19531b509e6ac"
03:03:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.247834 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37c86a9a5fe2437a9cadff003eddc10d" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83"
03:03:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.247948 17591 raft_consensus.cc:3037] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Advancing to term 1
03:03:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.248085 17732 raft_consensus.cc:3037] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Advancing to term 1
03:03:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.250352 17591 raft_consensus.cc:2454] T 37c86a9a5fe2437a9cadff003eddc10d P 1910a86c11064db0b0a19531b509e6ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 1.
03:03:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.250509 17732 raft_consensus.cc:2454] T 37c86a9a5fe2437a9cadff003eddc10d P 9c46998665694072ba111cab69a45c83 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 1.
03:03:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.250753 17801 leader_election.cc:302] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, e92d615261964bcf99f19c568f42c5cd; no voters: 
03:03:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.251089 17956 raft_consensus.cc:2781] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Leader election won for term 1
03:03:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.251550 17956 raft_consensus.cc:686] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [term 1 LEADER]: Becoming Leader. State: Replica: e92d615261964bcf99f19c568f42c5cd, State: Running, Role: LEADER
03:03:25.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.252074 17956 consensus_queue.cc:237] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.257599 17388 catalog_manager.cc:5495] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd reported cstate change: term changed from 0 to 1, leader changed from <none> to e92d615261964bcf99f19c568f42c5cd (127.16.200.67). New cstate: current_term: 1 leader_uuid: "e92d615261964bcf99f19c568f42c5cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } health_report { overall_health: HEALTHY } } }
03:03:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.282155 17956 raft_consensus.cc:482] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.282379 17956 raft_consensus.cc:504] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.282971 17956 leader_election.cc:288] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499), 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.283581 17591 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0cfb25a239cb45979a3dc2d7cb7f5422" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1910a86c11064db0b0a19531b509e6ac" is_pre_election: true
03:03:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.283599 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0cfb25a239cb45979a3dc2d7cb7f5422" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83" is_pre_election: true
03:03:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.283809 17591 raft_consensus.cc:2454] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 0.
03:03:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.283866 17732 raft_consensus.cc:2454] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 0.
03:03:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.284258 17801 leader_election.cc:302] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, e92d615261964bcf99f19c568f42c5cd; no voters: 
03:03:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.284590 17956 raft_consensus.cc:2781] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Leader pre-election won for term 1
03:03:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.284693 17956 raft_consensus.cc:482] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.284837 17956 raft_consensus.cc:3037] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Advancing to term 1
03:03:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.286669 17956 raft_consensus.cc:504] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.287241 17956 leader_election.cc:288] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 election: Requested vote from peers 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499), 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.287802 17591 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0cfb25a239cb45979a3dc2d7cb7f5422" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1910a86c11064db0b0a19531b509e6ac"
03:03:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.287822 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0cfb25a239cb45979a3dc2d7cb7f5422" candidate_uuid: "e92d615261964bcf99f19c568f42c5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83"
03:03:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.287994 17591 raft_consensus.cc:3037] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Advancing to term 1
03:03:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.288065 17732 raft_consensus.cc:3037] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Advancing to term 1
03:03:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.289911 17591 raft_consensus.cc:2454] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 1.
03:03:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.290066 17732 raft_consensus.cc:2454] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e92d615261964bcf99f19c568f42c5cd in term 1.
03:03:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.290355 17801 leader_election.cc:302] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, e92d615261964bcf99f19c568f42c5cd; no voters: 
03:03:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.290647 17956 raft_consensus.cc:2781] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Leader election won for term 1
03:03:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.290819 17956 raft_consensus.cc:686] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [term 1 LEADER]: Becoming Leader. State: Replica: e92d615261964bcf99f19c568f42c5cd, State: Running, Role: LEADER
03:03:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.291110 17956 consensus_queue.cc:237] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.294114 17388 catalog_manager.cc:5495] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd reported cstate change: term changed from 0 to 1, leader changed from <none> to e92d615261964bcf99f19c568f42c5cd (127.16.200.67). New cstate: current_term: 1 leader_uuid: "e92d615261964bcf99f19c568f42c5cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } health_report { overall_health: HEALTHY } } }
03:03:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.325276 17639 tablet.cc:2369] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 1910a86c11064db0b0a19531b509e6ac: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.333534 17388 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56126:
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
03:03:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:03:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.334671 17388 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:03:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.346459 17841 tablet_service.cc:1467] Processing CreateTablet for tablet c7015562712c447abcf4ef2ecdf06c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:03:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.347465 17841 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c7015562712c447abcf4ef2ecdf06c14. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.347047 17711 tablet_service.cc:1467] Processing CreateTablet for tablet c7015562712c447abcf4ef2ecdf06c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:03:25.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.347929 17711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c7015562712c447abcf4ef2ecdf06c14. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.348860 17565 tablet_service.cc:1467] Processing CreateTablet for tablet c7015562712c447abcf4ef2ecdf06c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:03:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.349691 17565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c7015562712c447abcf4ef2ecdf06c14. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.352480 17951 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: Bootstrap starting.
03:03:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.353802 17951 tablet_bootstrap.cc:654] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: Neither blocks nor log segments found. Creating new log.
03:03:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.354490 17949 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: Bootstrap starting.
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.356005 17951 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: No bootstrap required, opened a new log
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.356189 17949 tablet_bootstrap.cc:654] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: Neither blocks nor log segments found. Creating new log.
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.356226 17951 ts_tablet_manager.cc:1397] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: Time spent bootstrapping tablet: real 0.004s	user 0.003s	sys 0.000s
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.357057 17951 raft_consensus.cc:348] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.357368 17951 raft_consensus.cc:374] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.357445 17951 raft_consensus.cc:729] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e92d615261964bcf99f19c568f42c5cd, State: Initialized, Role: FOLLOWER
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.357700 17951 consensus_queue.cc:260] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.358264 17951 ts_tablet_manager.cc:1428] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.000s
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.358186 17956 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.358464 17956 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.358532 17949 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: No bootstrap required, opened a new log
03:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.358681 17949 ts_tablet_manager.cc:1397] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: Time spent bootstrapping tablet: real 0.004s	user 0.003s	sys 0.000s
03:03:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.359380 17949 raft_consensus.cc:348] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.359760 17949 raft_consensus.cc:374] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.359865 17949 raft_consensus.cc:729] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c46998665694072ba111cab69a45c83, State: Initialized, Role: FOLLOWER
03:03:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.360121 17949 consensus_queue.cc:260] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.360651 17949 ts_tablet_manager.cc:1428] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
03:03:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.360580 17954 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.361039 17954 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.366356 17948 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: Bootstrap starting.
03:03:25.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.367827 17948 tablet_bootstrap.cc:654] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: Neither blocks nor log segments found. Creating new log.
03:03:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.370064 17948 tablet_bootstrap.cc:492] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: No bootstrap required, opened a new log
03:03:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.370254 17948 ts_tablet_manager.cc:1397] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: Time spent bootstrapping tablet: real 0.004s	user 0.001s	sys 0.002s
03:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.370916 17948 raft_consensus.cc:348] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.371198 17948 raft_consensus.cc:374] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.371294 17948 raft_consensus.cc:729] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1910a86c11064db0b0a19531b509e6ac, State: Initialized, Role: FOLLOWER
03:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.371559 17948 consensus_queue.cc:260] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.372118 17948 ts_tablet_manager.cc:1428] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.000s
03:03:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.372078 17955 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.373049 17955 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.389243 17781 tablet.cc:2369] T 0cfb25a239cb45979a3dc2d7cb7f5422 P 9c46998665694072ba111cab69a45c83: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.412340 17955 raft_consensus.cc:482] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.412577 17955 raft_consensus.cc:504] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.414178 17955 leader_election.cc:288] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815), e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.422892 17877 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c7015562712c447abcf4ef2ecdf06c14" candidate_uuid: "1910a86c11064db0b0a19531b509e6ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e92d615261964bcf99f19c568f42c5cd" is_pre_election: true
03:03:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.423360 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c7015562712c447abcf4ef2ecdf06c14" candidate_uuid: "1910a86c11064db0b0a19531b509e6ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83" is_pre_election: true
03:03:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.423389 17877 raft_consensus.cc:2454] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1910a86c11064db0b0a19531b509e6ac in term 0.
03:03:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.423605 17732 raft_consensus.cc:2454] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1910a86c11064db0b0a19531b509e6ac in term 0.
03:03:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.424139 17513 leader_election.cc:302] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, 9c46998665694072ba111cab69a45c83, e92d615261964bcf99f19c568f42c5cd; no voters: 
03:03:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.424651 17955 raft_consensus.cc:2781] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Leader pre-election won for term 1
03:03:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.424796 17955 raft_consensus.cc:482] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.424898 17955 raft_consensus.cc:3037] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 0 FOLLOWER]: Advancing to term 1
03:03:25.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.426880 17955 raft_consensus.cc:504] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.427462 17955 leader_election.cc:288] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [CANDIDATE]: Term 1 election: Requested vote from peers 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815), e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.427968 17732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c7015562712c447abcf4ef2ecdf06c14" candidate_uuid: "1910a86c11064db0b0a19531b509e6ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c46998665694072ba111cab69a45c83"
03:03:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.428066 17877 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c7015562712c447abcf4ef2ecdf06c14" candidate_uuid: "1910a86c11064db0b0a19531b509e6ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e92d615261964bcf99f19c568f42c5cd"
03:03:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.428180 17732 raft_consensus.cc:3037] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 0 FOLLOWER]: Advancing to term 1
03:03:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.428254 17877 raft_consensus.cc:3037] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 0 FOLLOWER]: Advancing to term 1
03:03:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.430444 17732 raft_consensus.cc:2454] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1910a86c11064db0b0a19531b509e6ac in term 1.
03:03:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.430701 17877 raft_consensus.cc:2454] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1910a86c11064db0b0a19531b509e6ac in term 1.
03:03:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.430984 17515 leader_election.cc:302] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1910a86c11064db0b0a19531b509e6ac, 9c46998665694072ba111cab69a45c83; no voters: 
03:03:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.431327 17955 raft_consensus.cc:2781] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 1 FOLLOWER]: Leader election won for term 1
03:03:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.431828 17955 raft_consensus.cc:686] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [term 1 LEADER]: Becoming Leader. State: Replica: 1910a86c11064db0b0a19531b509e6ac, State: Running, Role: LEADER
03:03:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.432361 17955 consensus_queue.cc:237] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } }
03:03:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.441123 17976 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: TxnStatusTablet state changed. Reason: New leader 1910a86c11064db0b0a19531b509e6ac. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.441501 17976 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: This TxnStatusTablet replica's current role is: LEADER
03:03:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.441207 17388 catalog_manager.cc:5495] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 1910a86c11064db0b0a19531b509e6ac (127.16.200.65). New cstate: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } health_report { overall_health: UNKNOWN } } }
03:03:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.442313 17978 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
03:03:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.442595 17978 txn_status_manager.cc:929] Loading transaction status metadata into memory...
03:03:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.491710 17877 raft_consensus.cc:1261] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd [term 1 FOLLOWER]: Refusing update from remote peer 1910a86c11064db0b0a19531b509e6ac: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.491942 17732 raft_consensus.cc:1261] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83 [term 1 FOLLOWER]: Refusing update from remote peer 1910a86c11064db0b0a19531b509e6ac: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.492789 17976 consensus_queue.cc:1035] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.493108 17955 consensus_queue.cc:1035] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.507193 17954 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: TxnStatusTablet state changed. Reason: New leader 1910a86c11064db0b0a19531b509e6ac. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.507483 17954 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.509279 17956 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: TxnStatusTablet state changed. Reason: New leader 1910a86c11064db0b0a19531b509e6ac. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.509563 17956 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.510363 17981 mvcc.cc:205] Tried to move back new op lower bound from 7298763184084287488 to 7298763183854657536. Current Snapshot: MvccSnapshot[applied={T|T < 7298763184084287488}]
03:03:25.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.512787 17982 mvcc.cc:205] Tried to move back new op lower bound from 7298763184084287488 to 7298763183854657536. Current Snapshot: MvccSnapshot[applied={T|T < 7298763184084287488}]
03:03:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.513362 17955 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.513659 17955 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: This TxnStatusTablet replica's current role is: LEADER
03:03:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.513919 17955 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.514146 17955 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 1910a86c11064db0b0a19531b509e6ac: This TxnStatusTablet replica's current role is: LEADER
03:03:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.515796 17983 mvcc.cc:205] Tried to move back new op lower bound from 7298763184084287488 to 7298763183854657536. Current Snapshot: MvccSnapshot[applied={T|T < 7298763184084287488}]
03:03:25.526 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:03:25.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.525970 17954 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.526217 17954 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P 9c46998665694072ba111cab69a45c83: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.531642 17956 tablet_replica.cc:439] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1910a86c11064db0b0a19531b509e6ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 } } peers { permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 } } peers { permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" member_type: VOTER last_known_addr { host: "127.16.200.67" port: 39839 } } }
03:03:25.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.531922 17956 tablet_replica.cc:442] T c7015562712c447abcf4ef2ecdf06c14 P e92d615261964bcf99f19c568f42c5cd: This TxnStatusTablet replica's current role is: FOLLOWER
03:03:25.533 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ecbd5643117542d893822cc55e271228
03:03:25.533 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.124:40167
03:03:25.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

03:03:25.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.534785 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17355
03:03:25.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.536234 17429 generic_service.cc:196] Checking for leaks (request via RPC)
03:03:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.704468 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17355
03:03:25.725 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.200.124:40167(127.16.200.124:40167)] handling channelInactive
03:03:25.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state READY due to: connection closed
03:03:25.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
03:03:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.741278 17956 consensus_queue.cc:1035] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.746769 17778 heartbeater.cc:644] Failed to heartbeat to 127.16.200.124:40167 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:25.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.749426 17922 heartbeater.cc:644] Failed to heartbeat to 127.16.200.124:40167 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.750391 17956 consensus_queue.cc:1035] T 0cfb25a239cb45979a3dc2d7cb7f5422 P e92d615261964bcf99f19c568f42c5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:25.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:25.755285 17636 heartbeater.cc:644] Failed to heartbeat to 127.16.200.124:40167 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.800309 17960 consensus_queue.cc:1035] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c46998665694072ba111cab69a45c83" member_type: VOTER last_known_addr { host: "127.16.200.66" port: 38815 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:25.808249 17960 consensus_queue.cc:1035] T 37c86a9a5fe2437a9cadff003eddc10d P e92d615261964bcf99f19c568f42c5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" member_type: VOTER last_known_addr { host: "127.16.200.65" port: 36499 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.004594 17997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.004817 17997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.004855 17996 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.005105 17996 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.006718 17316 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d693881715e435abf6ea251c71c8288" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" is_pre_election: true
03:03:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.006968 17316 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d693881715e435abf6ea251c71c8288 in term 1.
03:03:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.007053 17997 leader_election.cc:288] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.007947 17236 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288" is_pre_election: true
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008057 17212 leader_election.cc:302] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 2d693881715e435abf6ea251c71c8288; no voters: 
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008240 17236 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0cafa7f6fbec485b99c2ecdd6c8af1eb in term 1.
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008359 17996 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008494 17997 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008611 17997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008698 17997 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 1 FOLLOWER]: Advancing to term 2
03:03:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.008810 17295 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 2d693881715e435abf6ea251c71c8288; no voters: 
03:03:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.009334 17996 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Leader pre-election won for term 2
03:03:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.009388 17213 leader_election.cc:334] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.009557 17996 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Starting leader election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.009649 17996 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 1 FOLLOWER]: Advancing to term 2
03:03:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.010381 17294 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.011368 17997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.011965 17997 leader_election.cc:288] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 election: Requested vote from peers 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.011958 17996 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.012359 17316 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d693881715e435abf6ea251c71c8288" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.012632 17316 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 2d693881715e435abf6ea251c71c8288 in current term 2: Already voted for candidate 0cafa7f6fbec485b99c2ecdd6c8af1eb in this term.
03:03:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.013016 17236 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288"
03:03:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.013017 17996 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 election: Requested vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.013309 17236 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 0cafa7f6fbec485b99c2ecdd6c8af1eb in current term 2: Already voted for candidate 2d693881715e435abf6ea251c71c8288 in this term.
03:03:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.013487 17213 leader_election.cc:334] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.013630 17213 leader_election.cc:302] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2d693881715e435abf6ea251c71c8288; no voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 1c105f56a90947ff95a026f305fc5904
03:03:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.013917 17997 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:03:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.014664 17294 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.014784 17294 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.015008 17996 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:03:26.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.033 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.033 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.033 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.16.200.124:40167(127.16.200.124:40167) for tablet Kudu Master: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.073 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.095 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.114 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.115 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.234 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] sending RPC to server master-127.16.200.124:40167, [1ms] received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [200ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1928621557(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@51bba440 -> (continuation of Deferred@294110821 after retry RPC after error@2119607108) -> (continuation of Deferred@790422206 after retry RPC after error@587848377) -> (continuation of Deferred@210932819 after retry RPC after error@1200966150) -> (continuation of Deferred@257182040 after retry RPC after error@1650820633) -> (continuation of Deferred@2051909130 after retry RPC after error@400885311) -> (continuation of Deferred@978748178 after retry RPC after error@1959632655), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@63270a6a -> (continuation of Deferred@294110821 after retry RPC after error@2119607108) -> (continuation of Deferred@790422206 after retry RPC after error@587848377) -> (continuation of Deferred@210932819 after retry RPC after error@1200966150) -> (continuation of Deferred@257182040 after retry RPC after error@1650820633) -> (continuation of Deferred@2051909130 after retry RPC after error@400885311) -> (continuation of Deferred@978748178 after retry RPC after error@1959632655)))
03:03:26.235 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] sending RPC to server master-127.16.200.124:40167, [1ms] received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [200ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1928621557(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@51bba440 -> (continuation of Deferred@294110821 after retry RPC after error@2119607108) -> (continuation of Deferred@790422206 after retry RPC after error@587848377) -> (continuation of Deferred@210932819 after retry RPC after error@1200966150) -> (continuation of Deferred@257182040 after retry RPC after error@1650820633) -> (continuation of Deferred@2051909130 after retry RPC after error@400885311) -> (continuation of Deferred@978748178 after retry RPC after error@1959632655), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@63270a6a -> (continuation of Deferred@294110821 after retry RPC after error@2119607108) -> (continuation of Deferred@790422206 after retry RPC after error@587848377) -> (continuation of Deferred@210932819 after retry RPC after error@1200966150) -> (continuation of Deferred@257182040 after retry RPC after error@1650820633) -> (continuation of Deferred@2051909130 after retry RPC after error@400885311) -> (continuation of Deferred@978748178 after retry RPC after error@1959632655)))
03:03:26.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.254 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.295 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.374 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.374329 17996 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.374550 17996 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.375195 17996 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.375671 17236 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288" is_pre_election: true
03:03:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.375977 17236 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0cafa7f6fbec485b99c2ecdd6c8af1eb in term 2.
03:03:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.376420 17295 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 2d693881715e435abf6ea251c71c8288; no voters: 
03:03:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.376725 17996 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Leader pre-election won for term 3
03:03:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.376684 17294 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.376825 17996 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:26.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.376948 17996 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 2 FOLLOWER]: Advancing to term 3
03:03:26.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.378760 17996 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.379305 17996 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 election: Requested vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.379956 17236 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288"
03:03:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.380175 17236 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 2 FOLLOWER]: Advancing to term 3
03:03:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.380703 17294 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.383018 17236 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0cafa7f6fbec485b99c2ecdd6c8af1eb in term 3.
03:03:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.383543 17295 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 2d693881715e435abf6ea251c71c8288; no voters: 1c105f56a90947ff95a026f305fc5904
03:03:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.383801 17996 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Leader election won for term 3
03:03:26.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.384235 17996 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 LEADER]: Becoming Leader. State: Replica: 0cafa7f6fbec485b99c2ecdd6c8af1eb, State: Running, Role: LEADER
03:03:26.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.384697 17996 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.386121 18003 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0cafa7f6fbec485b99c2ecdd6c8af1eb. Latest consensus state: current_term: 3 leader_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.386380 18003 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: LEADER
03:03:26.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.386965 18004 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:03:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.389348 18004 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]
03:03:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.389832 18004 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]
03:03:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.391424 18004 tablet_loader.cc:96] loaded metadata for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.391748 18004 tablet_loader.cc:96] loaded metadata for tablet 37c86a9a5fe2437a9cadff003eddc10d (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.392005 18004 tablet_loader.cc:96] loaded metadata for tablet c7015562712c447abcf4ef2ecdf06c14 (table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5])
03:03:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.392206 18004 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:03:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.393077 18004 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.393157 18004 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:03:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.394233 18004 catalog_manager.cc:1504] Loading token signing keys...
03:03:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.395030 18004 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Loaded TSK: 0
03:03:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.395825 18004 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:03:26.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:26.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:26.415 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:03:26.416 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=164), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, deferred=Deferred@1275293327(state=PENDING, result=null, callback=(continuation of Deferred@215962802 after retry RPC after error@1691801184) -> (continuation of Deferred@1214877040 after retry RPC after error@533325284) -> (continuation of Deferred@1774536811 after retry RPC after error@997385957) -> (continuation of Deferred@1064627958 after retry RPC after error@1866648121) -> (continuation of Deferred@738673066 after retry RPC after error@15650561) -> (continuation of Deferred@410354356 after retry RPC after error@1637847169), errback=(continuation of Deferred@215962802 after retry RPC after error@1691801184) -> (continuation of Deferred@1214877040 after retry RPC after error@533325284) -> (continuation of Deferred@1774536811 after retry RPC after error@997385957) -> (continuation of Deferred@1064627958 after retry RPC after error@1866648121) -> (continuation of Deferred@738673066 after retry RPC after error@15650561) -> (continuation of Deferred@410354356 after retry RPC after error@1637847169))) after lookup
03:03:26.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.481070 17307 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55694:
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:03:26.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.488869 17307 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:03:26.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.497341 17236 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Refusing update from remote peer 0cafa7f6fbec485b99c2ecdd6c8af1eb: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
03:03:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.498140 18003 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
03:03:26.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.500013 17997 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0cafa7f6fbec485b99c2ecdd6c8af1eb. Latest consensus state: current_term: 3 leader_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:26.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.500315 17997 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: FOLLOWER
03:03:26.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.505692 18009 meta_cache.cc:302] tablet c7015562712c447abcf4ef2ecdf06c14: replica 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499) has failed: Timed out: connection negotiation to 127.16.200.65:36499 for RPC CoordinateTransaction timed out after 0.005s (ON_OUTBOUND_QUEUE)
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.509133 17338 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:55684 (request call id 15) took 91 ms (client timeout 85 ms). Trace:
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.509646 17294 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb -> Peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Couldn't send request to peer 1c105f56a90947ff95a026f305fc5904. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:26.509332 17338 rpcz_store.cc:269] 0620 03:03:26.417456 (+     0us) service_pool.cc:170] Inserting onto call queue
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:26.417558 (+   102us) service_pool.cc:227] Handling call
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:26.509050 (+ 91492us) inbound_call.cc:173] Queueing success response
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":638,"init.queue_time_us":552,"init.run_cpu_time_us":8510,"init.run_wall_time_us":75456,"thread_start_us":1706,"threads_started":5}
03:03:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.510528 17997 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:26.510 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: Failed to write to server: (no server available): CoordinateTransaction(txn_status_tablet_id: "c7015562712c447abcf4ef2ecdf06c14" op { type: KEEP_TXN_ALIVE txn_id: 0 user: "slave" }) passed its deadline: Timed out: connection negotiation to 127.16.200.65:36499 for RPC CoordinateTransaction timed out after 0.005s (ON_OUTBOUND_QUEUE)
03:03:26.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.510802 17997 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: FOLLOWER
03:03:26.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.512182 18021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:26.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:26.512544 18021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: LEADER
03:03:27.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:27.040589 17294 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb -> Peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Couldn't send request to peer 1c105f56a90947ff95a026f305fc5904. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:03:27.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:27.519254 17294 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111) [suppressed 18 similar messages]
03:03:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:27.520325 17294 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb -> Peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Couldn't send request to peer 1c105f56a90947ff95a026f305fc5904. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:03:27.732 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 0cfb25a239cb45979a3dc2d7cb7f5422 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
03:03:27.732 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 37c86a9a5fe2437a9cadff003eddc10d for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
03:03:27.732 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=0cfb25a239cb45979a3dc2d7cb7f5422@[1910a86c11064db0b0a19531b509e6ac(127.16.200.65:36499),9c46998665694072ba111cab69a45c83(127.16.200.66:38815),e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=37c86a9a5fe2437a9cadff003eddc10d@[1910a86c11064db0b0a19531b509e6ac(127.16.200.65:36499),9c46998665694072ba111cab69a45c83(127.16.200.66:38815),e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
03:03:27.733 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC GetTableLocations: received response from server master-127.16.200.125:33579: OK, deferred=null) after lookup
03:03:27.733 [DEBUG - kudu-nio-0] (Connection.java:748) [peer e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)] connecting to peer
03:03:27.734 [DEBUG - kudu-nio-0] (Connection.java:757) [peer e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)] Successfully connected to peer
03:03:27.748 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0f55304b, L:/127.0.0.1:38926 - R:/127.16.200.67:39839] using TOKEN/null
03:03:27.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:27.752756 17859 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.16.200.124:40167)
03:03:27.797 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.200.124:40167
03:03:27.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:27.799069 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/logs
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.124:40167
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.124
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35103
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:03:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:03:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:27.811432 17780 heartbeater.cc:503] Master 127.16.200.125:33579 was elected leader, sending a full tablet report...
03:03:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:27.818070 17638 heartbeater.cc:503] Master 127.16.200.125:33579 was elected leader, sending a full tablet report...
03:03:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.053284 17294 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb -> Peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Couldn't send request to peer 1c105f56a90947ff95a026f305fc5904. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:03:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.080787 17196 master_runner.cc:186] Error getting master registration for 127.16.200.124:40167: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111), 
03:03:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.080961 17196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.140803 18045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.141129 18045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.141253 18045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.148640 18045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.148793 18045 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:03:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.148852 18045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.148926 18045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.148985 18045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.149071 18045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.149132 18045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45407/jwks.json
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.159799 18045 master_runner.cc:386] Master server non-default flags:
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:03:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.124:40167
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.124
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35103
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/logs
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.161480 18045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.163470 18045 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.175916 18053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.175957 18054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.176667 18056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.177575 18045 server_base.cc:1034] running on GCE node
03:03:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.178285 18045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.179664 18045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.180862 18045 hybrid_clock.cc:648] HybridClock initialized: now 1781924608180812 us; error 42 us; skew 500 ppm
03:03:28.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.181166 18045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:28.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.187264 18045 webserver.cc:460] Webserver started at http://127.16.200.124:35103/ using document root <none> and password file <none>
03:03:28.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.187929 18045 fs_manager.cc:359] Metadata directory not provided
03:03:28.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.188028 18045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:28.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.193099 18045 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.001s
03:03:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.196388 18062 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.197822 18045 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.198020 18045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c105f56a90947ff95a026f305fc5904"
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.198982 18045 fs_report.cc:383] FS layout report
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/wal
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/data
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:28.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.223089 18045 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.223923 18045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.224177 18045 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.232205 17924 heartbeater.cc:503] Master 127.16.200.125:33579 was elected leader, sending a full tablet report...
03:03:28.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.248271 18045 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.124:40167
03:03:28.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.248353 18124 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.124:40167 every 8 connection(s)
03:03:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.249722 18045 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-2/data/info.pb
03:03:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.256248 18125 sys_catalog.cc:263] Verifying existing consensus state
03:03:28.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.258052 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 18045
03:03:28.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
03:03:28.258 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.125:33579
03:03:28.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

03:03:28.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.258857 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17274
03:03:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.259873 18125 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Bootstrap starting.
03:03:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.260169 17348 generic_service.cc:196] Checking for leaks (request via RPC)
03:03:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.278245 18125 log.cc:824] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Log is configured to *not* fsync() on all Append() calls
03:03:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.292465 18125 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.293012 18125 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Bootstrap complete.
03:03:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.302662 18125 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.303678 18125 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c105f56a90947ff95a026f305fc5904, State: Initialized, Role: FOLLOWER
03:03:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.304327 18125 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.305246 18129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.305639 18129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: FOLLOWER
03:03:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.306376 18125 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: configured and running, proceeding with master startup.
03:03:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.323956 18140 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.324082 18140 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: loading cluster ID for follower catalog manager: success
03:03:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.332190 18140 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: acquiring CA information for follower catalog manager: success
03:03:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.338546 18140 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.408499 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17274
03:03:28.440 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.200.125:33579(127.16.200.125:33579)] handling channelInactive
03:03:28.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state READY due to: connection closed
03:03:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.441537 18143 negotiation.cc:337] Failed RPC negotiation. Trace:
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.349074 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.16.200.125:33579 (local address 127.0.0.1:55706)
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.349309 (+   235us) negotiation.cc:107] Waiting for socket to connect
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.349326 (+    17us) client_negotiation.cc:174] Beginning negotiation
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.349429 (+   103us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.440201 (+ 90772us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: BlockingRecv error: recv got EOF from 127.16.200.125:33579 (error 108)
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":67}
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.442512 17211 connection.cc:535] server connection from 127.0.0.1:33558 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.442965 18146 negotiation.cc:337] Failed RPC negotiation. Trace:
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.409233 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56660 (local address 127.16.200.124:40167)
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410369 (+  1136us) server_negotiation.cc:206] Beginning negotiation
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410378 (+     9us) server_negotiation.cc:399] Waiting for connection header
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410415 (+    37us) server_negotiation.cc:407] Connection header received
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410525 (+   110us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410534 (+     9us) server_negotiation.cc:461] Received NEGOTIATE request from client
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410815 (+   281us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.410921 (+   106us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.442425 (+ 31504us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56660: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:03:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":926,"thread_start_us":1061,"threads_started":1}
03:03:28.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
03:03:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.442183 18045 master_runner.cc:186] Error getting master registration for 127.16.200.125:33579: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: BlockingRecv error: recv got EOF from 127.16.200.125:33579 (error 108), 
03:03:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.449133 18045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.590358 18129 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.590785 18129 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.593438 18129 leader_election.cc:288] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579)
03:03:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.594063 17236 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "2d693881715e435abf6ea251c71c8288" is_pre_election: true
03:03:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.594326 18064 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.596064 18064 leader_election.cc:334] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.596215 18064 leader_election.cc:302] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c105f56a90947ff95a026f305fc5904; no voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 2d693881715e435abf6ea251c71c8288
03:03:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.596602 18129 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:03:28.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.612 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.16.200.125:33579(127.16.200.125:33579) for tablet Kudu Master: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:28.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.632 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.200.124:40167(127.16.200.124:40167)] Successfully connected to peer
03:03:28.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.646 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x09a2ed22, L:/127.0.0.1:56676 - R:/127.16.200.124:40167] using TOKEN/null
03:03:28.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.693 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.716017 18148 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 0cafa7f6fbec485b99c2ecdd6c8af1eb)
03:03:28.716 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.716266 18148 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.717036 18148 leader_election.cc:288] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.718670 17212 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111) [suppressed 9 similar messages]
03:03:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.720384 17212 leader_election.cc:334] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.726627 18090 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d693881715e435abf6ea251c71c8288" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1c105f56a90947ff95a026f305fc5904" is_pre_election: true
03:03:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.727123 18090 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d693881715e435abf6ea251c71c8288 in term 1.
03:03:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.727687 17213 leader_election.cc:302] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288; no voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb
03:03:28.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.728029 18148 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Leader pre-election won for term 4
03:03:28.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.728153 18148 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 0cafa7f6fbec485b99c2ecdd6c8af1eb)
03:03:28.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.728263 18148 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 3 FOLLOWER]: Advancing to term 4
03:03:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.730225 18148 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.730818 18148 leader_election.cc:288] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 election: Requested vote from peers 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.731376 18090 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d693881715e435abf6ea251c71c8288" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1c105f56a90947ff95a026f305fc5904"
03:03:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.731592 18090 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 1 FOLLOWER]: Advancing to term 4
03:03:28.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.732353 17212 leader_election.cc:334] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111)
03:03:28.733 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.734 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.734 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.737768 18090 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2d693881715e435abf6ea251c71c8288 in term 4.
03:03:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.738250 17213 leader_election.cc:302] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288; no voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb
03:03:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.738622 18148 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 4 FOLLOWER]: Leader election won for term 4
03:03:28.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.739452 18148 raft_consensus.cc:686] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [term 4 LEADER]: Becoming Leader. State: Replica: 2d693881715e435abf6ea251c71c8288, State: Running, Role: LEADER
03:03:28.739 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.739970 18148 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.741799 18152 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2d693881715e435abf6ea251c71c8288. Latest consensus state: current_term: 4 leader_uuid: "2d693881715e435abf6ea251c71c8288" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.742405 18152 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: LEADER
03:03:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.743255 18154 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:03:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.746061 18154 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]
03:03:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.746675 18154 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]
03:03:28.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.748631 18154 tablet_loader.cc:96] loaded metadata for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.748976 18154 tablet_loader.cc:96] loaded metadata for tablet 37c86a9a5fe2437a9cadff003eddc10d (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.749231 18154 tablet_loader.cc:96] loaded metadata for tablet c7015562712c447abcf4ef2ecdf06c14 (table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5])
03:03:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.749445 18154 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:03:28.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.750232 18154 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:28.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.750314 18154 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:03:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.751446 18154 catalog_manager.cc:1504] Loading token signing keys...
03:03:28.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.753387 18154 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288: Loaded TSK: 0
03:03:28.754 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.754513 18154 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:03:28.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.818104 17778 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.821909 18080 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9c46998665694072ba111cab69a45c83" instance_seqno: 1781924604341149) as {username='slave'} at 127.16.200.66:44681; Asking this server to re-register.
03:03:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.822592 17778 heartbeater.cc:464] Registering TS with master...
03:03:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.824338 17637 heartbeater.cc:503] Master 127.16.200.126:36191 was elected leader, sending a full tablet report...
03:03:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.824369 18080 ts_manager.cc:194] Registered new tserver with Master: 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.825109 17636 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.826428 18080 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" instance_seqno: 1781924603810001) as {username='slave'} at 127.16.200.65:51479; Asking this server to re-register.
03:03:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.826982 17636 heartbeater.cc:464] Registering TS with master...
03:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.828091 18080 ts_manager.cc:194] Registered new tserver with Master: 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499)
03:03:28.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:28.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.125:33579: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579
03:03:28.834 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:03:28.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.16.200.125:33579, [1ms] received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [105ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [129ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, deferred=Deferred@1709368075(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@48bd2485 -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@2a79170d -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266))) after lookup
03:03:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.836884 17212 consensus_peers.cc:487] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 -> Peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Couldn't send request to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:03:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.856860 18090 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Refusing update from remote peer 2d693881715e435abf6ea251c71c8288: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
03:03:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.858570 18152 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
03:03:28.866 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.16.200.125:33579, [1ms] received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [105ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [129ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [225ms] sending RPC to server master-127.16.200.126:36191, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1709368075(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@48bd2485 -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266) -> (continuation of Deferred@1296475153 after retry RPC@1325423173), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@2a79170d -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266) -> (continuation of Deferred@1296475153 after retry RPC@1325423173)))
03:03:28.867 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.16.200.125:33579, [1ms] received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [105ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [129ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: Network error: Failed to connect to peer master-127.16.200.125:33579(127.16.200.125:33579): Connection refused: /127.16.200.125:33579, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: OK, [225ms] sending RPC to server master-127.16.200.126:36191, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1709368075(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@48bd2485 -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266) -> (continuation of Deferred@1296475153 after retry RPC@1325423173), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@2a79170d -> (continuation of Deferred@317125784 after retry RPC after error@1767992254) -> (continuation of Deferred@468264738 after retry RPC after error@986821078) -> (continuation of Deferred@532074528 after retry RPC after error@2112382195) -> (continuation of Deferred@795515995 after retry RPC after error@402839393) -> (continuation of Deferred@823927697 after retry RPC after error@258803389) -> (continuation of Deferred@1948175183 after retry RPC after error@2049553266) -> (continuation of Deferred@1296475153 after retry RPC@1325423173)))
03:03:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.862493 17257 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40586 (request call id 22) took 25 ms (client timeout 25 ms). Trace:
03:03:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.862764 17257 rpcz_store.cc:269] 0620 03:03:28.837138 (+     0us) service_pool.cc:170] Inserting onto call queue
03:03:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.837226 (+    88us) service_pool.cc:227] Handling call
03:03:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:28.862410 (+ 25184us) inbound_call.cc:173] Queueing success response
03:03:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":239,"init.queue_time_us":460,"thread_start_us":1497,"threads_started":5}
03:03:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.882402 18129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2d693881715e435abf6ea251c71c8288. Latest consensus state: current_term: 4 leader_uuid: "2d693881715e435abf6ea251c71c8288" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.882702 18129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: FOLLOWER
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.882288 17225 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40616:
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.884536 18129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "2d693881715e435abf6ea251c71c8288" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.884800 18129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: FOLLOWER
03:03:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:28.884795 17225 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:03:28.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.889212 18170 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "2d693881715e435abf6ea251c71c8288" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.889616 18170 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: FOLLOWER
03:03:28.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.890988 18152 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "2d693881715e435abf6ea251c71c8288" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:28.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:28.891258 18152 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 [sys.catalog]: This master's current role is: LEADER
03:03:29.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:29.241518 17922 heartbeater.cc:347] Connected to a master server at 127.16.200.124:40167
03:03:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:29.242916 18080 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" instance_seqno: 1781924604963706) as {username='slave'} at 127.16.200.67:48783; Asking this server to re-register.
03:03:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:29.243608 17922 heartbeater.cc:464] Registering TS with master...
03:03:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:29.244891 18080 ts_manager.cc:194] Registered new tserver with Master: e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:29.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:29.308913 17212 consensus_peers.cc:487] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 -> Peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Couldn't send request to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:03:29.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:29.815042 17779 heartbeater.cc:503] Master 127.16.200.126:36191 was elected leader, sending a full tablet report...
03:03:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:29.845224 17212 consensus_peers.cc:487] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 -> Peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Couldn't send request to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:03:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.236661 17923 heartbeater.cc:503] Master 127.16.200.126:36191 was elected leader, sending a full tablet report...
03:03:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.410025 17212 consensus_peers.cc:487] T 00000000000000000000000000000000 P 2d693881715e435abf6ea251c71c8288 -> Peer 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579): Couldn't send request to peer 0cafa7f6fbec485b99c2ecdd6c8af1eb. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.125:33579: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:03:30.469 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.200.125:33579
03:03:30.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.470834 17185 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksldxbt/build/asan/bin/kudu
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/logs
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.125:33579
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.125
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39307
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:03:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.807771 18197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.808236 18197 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.808370 18197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816378 18197 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816541 18197 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816593 18197 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816679 18197 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816740 18197 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816826 18197 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.816916 18197 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45407/jwks.json
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.827286 18197 master_runner.cc:386] Master server non-default flags:
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.84:37185
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.125:33579
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45407/jwks.json
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.125
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39307
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/logs
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 02:00:34 UTC on ccycloud.quasar-ycjack.root.comops.site
03:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456918
03:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.828997 18197 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:30.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.831359 18197 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:03:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.844658 18205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.844733 18207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:30.844805 18204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:30.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.846130 18197 server_base.cc:1034] running on GCE node
03:03:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.846987 18197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.848562 18197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.849807 18197 hybrid_clock.cc:648] HybridClock initialized: now 1781924610849739 us; error 52 us; skew 500 ppm
03:03:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.850231 18197 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.852733 18197 webserver.cc:460] Webserver started at http://127.16.200.125:39307/ using document root <none> and password file <none>
03:03:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.853447 18197 fs_manager.cc:359] Metadata directory not provided
03:03:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.853610 18197 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:30.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.859478 18197 fs_manager.cc:709] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
03:03:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.863067 18213 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
03:03:30.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.864629 18197 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
03:03:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.864787 18197 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data,/tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 03:03:22 on dist-test-slave-214v"
03:03:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.865712 18197 fs_report.cc:383] FS layout report
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/wal
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/data
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.885598 18197 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
03:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.886386 18197 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.886637 18197 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.908090 18197 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.125:33579
03:03:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.908187 18274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.200.125:33579 every 8 connection(s)
03:03:30.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.909613 18197 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksldxbt/test-tmp/mini-kudu-cluster414022968894066891/master-1/data/info.pb
03:03:30.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.916333 18275 sys_catalog.cc:263] Verifying existing consensus state
03:03:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.918041 17185 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksldxbt/build/asan/bin/kudu as pid 18197
03:03:30.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
03:03:30.918 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.126:36191
03:03:30.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

03:03:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.918907 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17196
03:03:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.920045 17267 generic_service.cc:196] Checking for leaks (request via RPC)
03:03:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.920042 18275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Bootstrap starting.
03:03:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.938119 18275 log.cc:824] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Log is configured to *not* fsync() on all Append() calls
03:03:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.951505 18275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:30.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.952005 18275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: Bootstrap complete.
03:03:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.961351 18275 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.962334 18275 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0cafa7f6fbec485b99c2ecdd6c8af1eb, State: Initialized, Role: FOLLOWER
03:03:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.962994 18275 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.963830 18279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.964231 18279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:30.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.965181 18275 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: configured and running, proceeding with master startup.
03:03:30.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.978389 18290 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:30.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.978506 18290 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: loading cluster ID for follower catalog manager: success
03:03:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.982812 18290 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: acquiring CA information for follower catalog manager: success
03:03:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:30.985424 18290 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.062116 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17196
03:03:31.084 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.200.126:36191(127.16.200.126:36191)] handling channelInactive
03:03:31.084 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state READY due to: connection closed
03:03:31.084 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.16.200.126:36191(127.16.200.126:36191) for tablet Kudu Master: connection closed
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.085251 18292 negotiation.cc:337] Failed RPC negotiation. Trace:
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:30.973112 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.16.200.126:36191 (local address 127.0.0.1:40628)
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:30.974493 (+  1381us) negotiation.cc:107] Waiting for socket to connect
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:30.974535 (+    42us) client_negotiation.cc:174] Beginning negotiation
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:30.975126 (+   591us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:31.083742 (+108616us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: BlockingRecv error: recv got EOF from 127.16.200.126:36191 (error 108)
03:03:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":636,"thread_start_us":386,"threads_started":1}
03:03:31.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
03:03:31.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.085922 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: BlockingRecv error: recv got EOF from 127.16.200.126:36191 (error 108), 
03:03:31.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.125:33579(127.16.200.125:33579)] connecting to peer
03:03:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.092481 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:31.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.094 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.200.125:33579(127.16.200.125:33579)] Successfully connected to peer
03:03:31.110 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xad67bc2e, L:/127.0.0.1:56196 - R:/127.16.200.125:33579] using TOKEN/null
03:03:31.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.214 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.16.200.126:36191, [113ms] received response from server master-127.16.200.126:36191: Network error: connection closed, [113ms] delaying RPC due to: Network error: connection closed, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@35793600(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@562a43b5 -> (continuation of Deferred@1075145293 after retry RPC after error@1109528119) -> (continuation of Deferred@1363499161 after retry RPC after error@1442221807) -> (continuation of Deferred@1046139166 after retry RPC after error@1817916244) -> (continuation of Deferred@396439510 after retry RPC after error@1983994427) -> (continuation of Deferred@511420364 after retry RPC after error@722462404), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@35cea3ec -> (continuation of Deferred@1075145293 after retry RPC after error@1109528119) -> (continuation of Deferred@1363499161 after retry RPC after error@1442221807) -> (continuation of Deferred@1046139166 after retry RPC after error@1817916244) -> (continuation of Deferred@396439510 after retry RPC after error@1983994427) -> (continuation of Deferred@511420364 after retry RPC after error@722462404)))
03:03:31.214 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.16.200.126:36191, [113ms] received response from server master-127.16.200.126:36191: Network error: connection closed, [113ms] delaying RPC due to: Network error: connection closed, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@35793600(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@562a43b5 -> (continuation of Deferred@1075145293 after retry RPC after error@1109528119) -> (continuation of Deferred@1363499161 after retry RPC after error@1442221807) -> (continuation of Deferred@1046139166 after retry RPC after error@1817916244) -> (continuation of Deferred@396439510 after retry RPC after error@1983994427) -> (continuation of Deferred@511420364 after retry RPC after error@722462404), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@35cea3ec -> (continuation of Deferred@1075145293 after retry RPC after error@1109528119) -> (continuation of Deferred@1363499161 after retry RPC after error@1442221807) -> (continuation of Deferred@1046139166 after retry RPC after error@1817916244) -> (continuation of Deferred@396439510 after retry RPC after error@1983994427) -> (continuation of Deferred@511420364 after retry RPC after error@722462404)))
03:03:31.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.247272 17924 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:31.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.251570 18230 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e92d615261964bcf99f19c568f42c5cd" instance_seqno: 1781924604963706) as {username='slave'} at 127.16.200.67:47427; Asking this server to re-register.
03:03:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.252821 17924 heartbeater.cc:464] Registering TS with master...
03:03:31.253 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.254 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.255 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.254946 18230 ts_manager.cc:194] Registered new tserver with Master: e92d615261964bcf99f19c568f42c5cd (127.16.200.67:39839)
03:03:31.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.274 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.274 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.275 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.294 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.294 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.294 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.294 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.319936 18279 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.320472 18279 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:31.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.322610 18279 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.324412 18217 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:31.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.337638 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.344158 18090 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1c105f56a90947ff95a026f305fc5904" is_pre_election: true
03:03:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.345223 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.345669 18279 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:03:31.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.356 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.378 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.380796 18301 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 2d693881715e435abf6ea251c71c8288)
03:03:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.381191 18301 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.391151 18301 leader_election.cc:288] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579)
03:03:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.402374 18240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" is_pre_election: true
03:03:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.402755 18066 leader_election.cc:334] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.403005 18240 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1c105f56a90947ff95a026f305fc5904 in term 3.
03:03:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.403699 18064 leader_election.cc:302] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 1c105f56a90947ff95a026f305fc5904; no voters: 2d693881715e435abf6ea251c71c8288
03:03:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.404111 18301 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Leader pre-election won for term 5
03:03:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.404240 18301 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 2d693881715e435abf6ea251c71c8288)
03:03:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.404348 18301 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 4 FOLLOWER]: Advancing to term 5
03:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.406513 18301 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.407399 18301 leader_election.cc:288] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 election: Requested vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 0cafa7f6fbec485b99c2ecdd6c8af1eb (127.16.200.125:33579)
03:03:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.407752 18240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1c105f56a90947ff95a026f305fc5904" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb"
03:03:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.407970 18240 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 3 FOLLOWER]: Advancing to term 5
03:03:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.408931 18066 leader_election.cc:334] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.413692 18240 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1c105f56a90947ff95a026f305fc5904 in term 5.
03:03:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.414206 18064 leader_election.cc:302] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb, 1c105f56a90947ff95a026f305fc5904; no voters: 2d693881715e435abf6ea251c71c8288
03:03:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.414557 18301 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 5 FOLLOWER]: Leader election won for term 5
03:03:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.415164 18301 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [term 5 LEADER]: Becoming Leader. State: Replica: 1c105f56a90947ff95a026f305fc5904, State: Running, Role: LEADER
03:03:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.415647 18301 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 4.13, Last appended by leader: 13, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.417500 18305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1c105f56a90947ff95a026f305fc5904. Latest consensus state: current_term: 5 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.417819 18305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: LEADER
03:03:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.418568 18307 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:03:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.421008 18307 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5]
03:03:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.421500 18307 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228]
03:03:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.423481 18307 tablet_loader.cc:96] loaded metadata for tablet 0cfb25a239cb45979a3dc2d7cb7f5422 (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.423852 18307 tablet_loader.cc:96] loaded metadata for tablet 37c86a9a5fe2437a9cadff003eddc10d (table txn_manager_fallback_rolling [id=ecbd5643117542d893822cc55e271228])
03:03:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.424099 18307 tablet_loader.cc:96] loaded metadata for tablet c7015562712c447abcf4ef2ecdf06c14 (table kudu_system.kudu_transactions [id=5e68ddd14e6f4861a9e44eb0fbd4d0f5])
03:03:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.424312 18307 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:03:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.425176 18307 catalog_manager.cc:1259] Loaded cluster ID: 1d8ec70146c84f82ab5f478adc1ceb3b
03:03:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.425256 18307 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:03:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.426308 18307 catalog_manager.cc:1504] Loading token signing keys...
03:03:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.427064 18307 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904: Loaded TSK: 0
03:03:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.427839 18307 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:03:31.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:31.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:31.473 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:03:31.474 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [145ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, deferred=Deferred@1035127565(state=PENDING, result=null, callback=(continuation of Deferred@1919470152 after retry RPC after error@1540951052) -> (continuation of Deferred@1757546040 after retry RPC after error@1132675726) -> (continuation of Deferred@2005789244 after retry RPC after error@1688674733) -> (continuation of Deferred@472651975 after retry RPC after error@1164099326) -> (continuation of Deferred@1562800228 after retry RPC after error@13735972) -> (continuation of Deferred@2085049049 after retry RPC after error@738363105) -> (continuation of Deferred@1120495189 after retry RPC after error@1314248336), errback=(continuation of Deferred@1919470152 after retry RPC after error@1540951052) -> (continuation of Deferred@1757546040 after retry RPC after error@1132675726) -> (continuation of Deferred@2005789244 after retry RPC after error@1688674733) -> (continuation of Deferred@472651975 after retry RPC after error@1164099326) -> (continuation of Deferred@1562800228 after retry RPC after error@13735972) -> (continuation of Deferred@2085049049 after retry RPC after error@738363105) -> (continuation of Deferred@1120495189 after retry RPC after error@1314248336))) after lookup
03:03:31.493 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=261), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [145ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] sending RPC to server master-127.16.200.124:40167, deferred=Deferred@1035127565(state=PENDING, result=null, callback=(continuation of Deferred@1919470152 after retry RPC after error@1540951052) -> (continuation of Deferred@1757546040 after retry RPC after error@1132675726) -> (continuation of Deferred@2005789244 after retry RPC after error@1688674733) -> (continuation of Deferred@472651975 after retry RPC after error@1164099326) -> (continuation of Deferred@1562800228 after retry RPC after error@13735972) -> (continuation of Deferred@2085049049 after retry RPC after error@738363105) -> (continuation of Deferred@1120495189 after retry RPC after error@1314248336) -> (continuation of Deferred@422197284 after retry RPC@247069978), errback=(continuation of Deferred@1919470152 after retry RPC after error@1540951052) -> (continuation of Deferred@1757546040 after retry RPC after error@1132675726) -> (continuation of Deferred@2005789244 after retry RPC after error@1688674733) -> (continuation of Deferred@472651975 after retry RPC after error@1164099326) -> (continuation of Deferred@1562800228 after retry RPC after error@13735972) -> (continuation of Deferred@2085049049 after retry RPC after error@738363105) -> (continuation of Deferred@1120495189 after retry RPC after error@1314248336) -> (continuation of Deferred@422197284 after retry RPC@247069978)))
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.504839 18110 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:56676 (request call id 21) took 29 ms (client timeout 6 ms). Trace:
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.505367 18066 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 -> Peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Couldn't send request to peer 2d693881715e435abf6ea251c71c8288. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.505124 18110 rpcz_store.cc:269] 0620 03:03:31.475533 (+     0us) service_pool.cc:170] Inserting onto call queue
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:31.475618 (+    85us) service_pool.cc:227] Handling call
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 03:03:31.504743 (+ 29125us) inbound_call.cc:173] Queueing success response
03:03:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":126,"init.queue_time_us":438,"thread_start_us":1247,"threads_started":5}
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.510895 18080 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56706:
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:03:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 628028108118 + 9223371408826672990 cannot be represented in type 'long'
03:03:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:31.513986 18080 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:03:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.514528 18240 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Refusing update from remote peer 1c105f56a90947ff95a026f305fc5904: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 14. (index mismatch)
03:03:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.515681 18305 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
03:03:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.521889 18279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1c105f56a90947ff95a026f305fc5904. Latest consensus state: current_term: 5 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.522243 18279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.522495 18319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.522846 18319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.523594 18319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.523814 18319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [sys.catalog]: This master's current role is: FOLLOWER
03:03:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.531628 18301 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "1c105f56a90947ff95a026f305fc5904" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } } }
03:03:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.532487 18301 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 [sys.catalog]: This master's current role is: LEADER
03:03:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.832010 17780 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.833329 18230 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9c46998665694072ba111cab69a45c83" instance_seqno: 1781924604341149) as {username='slave'} at 127.16.200.66:57769; Asking this server to re-register.
03:03:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.833933 17780 heartbeater.cc:464] Registering TS with master...
03:03:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.835531 18230 ts_manager.cc:194] Registered new tserver with Master: 9c46998665694072ba111cab69a45c83 (127.16.200.66:38815)
03:03:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.846189 17638 heartbeater.cc:347] Connected to a master server at 127.16.200.125:33579
03:03:31.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.847635 18230 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1910a86c11064db0b0a19531b509e6ac" instance_seqno: 1781924603810001) as {username='slave'} at 127.16.200.65:43177; Asking this server to re-register.
03:03:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.848169 17638 heartbeater.cc:464] Registering TS with master...
03:03:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:31.849339 18230 ts_manager.cc:194] Registered new tserver with Master: 1910a86c11064db0b0a19531b509e6ac (127.16.200.65:36499)
03:03:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:32.043599 18066 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 -> Peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Couldn't send request to peer 2d693881715e435abf6ea251c71c8288. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:03:32.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:32.528220 18066 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1c105f56a90947ff95a026f305fc5904 -> Peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Couldn't send request to peer 2d693881715e435abf6ea251c71c8288. Status: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:03:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #0 0x7f5253f51538 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28
03:03:32.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #1 0x9dd255 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99:38
03:03:32.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #2 0x7f5266fc1be2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/promise.h:57:16
03:03:32.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #3 0x7f5266fb27d4 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:458:48
03:03:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #4 0x7f526710716b in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:283:21
03:03:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #5 0x7f5267107d5b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::string const&, kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:226:3
03:03:32.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #6 0x7f526710c9cf in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager_service.cc:159:42
03:03:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #7 0x7f52611bdcc6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/build/asan/src/kudu/master/txn_manager.service.cc:271:13
03:03:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #8 0x7f52611bd9f5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/8/bits/std_function.h:297:2
03:03:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #9 0x7f525ab2e9be in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/8/bits/std_function.h:687:14
03:03:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #10 0x7f525ab2d79e in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:137:3
03:03:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #11 0x7f525ab33b25 in kudu::rpc::ServicePool::RunThread() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:231:15
03:03:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #12 0x7f525ab3454f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:92:5
03:03:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #13 0x7f525ab34378 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/8/bits/std_function.h:297:2
03:03:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #14 0x9edb7c in std::function<void ()>::operator()() const ../../../include/c++/8/bits/std_function.h:687:14
03:03:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #15 0x7f5254016eae in kudu::Thread::SuperviseThread(void*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694:3
03:03:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #16 0x7f52572f06da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
03:03:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #17 0x7f525028661e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12161e)
03:03:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
03:03:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28 in 
03:03:32.713 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.200.124:40167(127.16.200.124:40167)] handling channelInactive
03:03:32.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state READY due to: connection closed
03:03:32.713 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.16.200.124:40167(127.16.200.124:40167) for tablet Kudu Master: connection closed
03:03:32.732 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:32.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.833053 17778 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.838848 17636 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.910044 18333 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.910256 18333 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.910900 18333 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:32.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:32.913337 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:32.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:32.913568 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:32.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.913712 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:32.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:32.914029 18333 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:33.086 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.086 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.087 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.087 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.087 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.087 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.087 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.087 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.088 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.115 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.133 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.134 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.135 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.136 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.153 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.153 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.155 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.155 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.175 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.193 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.193 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.195 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.216 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.243708 17922 heartbeater.cc:503] Master 127.16.200.124:40167 was elected leader, sending a full tablet report...
03:03:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.245675 18333 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.245867 18333 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.246476 18333 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:33.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:33.247782 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:33.247977 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.248145 18217 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.248471 18333 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:33.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.275 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.351 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [125ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@654221397(state=PENDING, result=null, callback=(continuation of Deferred@883930505 after retry RPC after error@1804221319) -> (continuation of Deferred@1013812500 after retry RPC after error@1875013316) -> (continuation of Deferred@646483272 after retry RPC after error@484756509) -> (continuation of Deferred@1278343096 after retry RPC after error@813237428) -> (continuation of Deferred@1383677246 after retry RPC after error@1749522248) -> (continuation of Deferred@721957327 after retry RPC after error@1395444926) -> (continuation of Deferred@1210555176 after retry RPC after error@470722739), errback=(continuation of Deferred@883930505 after retry RPC after error@1804221319) -> (continuation of Deferred@1013812500 after retry RPC after error@1875013316) -> (continuation of Deferred@646483272 after retry RPC after error@484756509) -> (continuation of Deferred@1278343096 after retry RPC after error@813237428) -> (continuation of Deferred@1383677246 after retry RPC after error@1749522248) -> (continuation of Deferred@721957327 after retry RPC after error@1395444926) -> (continuation of Deferred@1210555176 after retry RPC after error@470722739)))
03:03:33.352 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:33.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.375 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.453 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.473 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.573 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.574 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.575 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.576 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1301021345(state=PENDING, result=null, callback=(continuation of Deferred@261283397 after retry RPC after error@518411388) -> (continuation of Deferred@382303779 after retry RPC after error@998109009) -> (continuation of Deferred@579108493 after retry RPC after error@242064149) -> (continuation of Deferred@1393083549 after retry RPC after error@131837642) -> (continuation of Deferred@103471274 after retry RPC after error@1694326345) -> (continuation of Deferred@1908691559 after retry RPC after error@724766571) -> (continuation of Deferred@2072800559 after retry RPC after error@1963817729), errback=(continuation of Deferred@261283397 after retry RPC after error@518411388) -> (continuation of Deferred@382303779 after retry RPC after error@998109009) -> (continuation of Deferred@579108493 after retry RPC after error@242064149) -> (continuation of Deferred@1393083549 after retry RPC after error@131837642) -> (continuation of Deferred@103471274 after retry RPC after error@1694326345) -> (continuation of Deferred@1908691559 after retry RPC after error@724766571) -> (continuation of Deferred@2072800559 after retry RPC after error@1963817729)))
03:03:33.577 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1301021345(state=PENDING, result=null, callback=(continuation of Deferred@261283397 after retry RPC after error@518411388) -> (continuation of Deferred@382303779 after retry RPC after error@998109009) -> (continuation of Deferred@579108493 after retry RPC after error@242064149) -> (continuation of Deferred@1393083549 after retry RPC after error@131837642) -> (continuation of Deferred@103471274 after retry RPC after error@1694326345) -> (continuation of Deferred@1908691559 after retry RPC after error@724766571) -> (continuation of Deferred@2072800559 after retry RPC after error@1963817729), errback=(continuation of Deferred@261283397 after retry RPC after error@518411388) -> (continuation of Deferred@382303779 after retry RPC after error@998109009) -> (continuation of Deferred@579108493 after retry RPC after error@242064149) -> (continuation of Deferred@1393083549 after retry RPC after error@131837642) -> (continuation of Deferred@103471274 after retry RPC after error@1694326345) -> (continuation of Deferred@1908691559 after retry RPC after error@724766571) -> (continuation of Deferred@2072800559 after retry RPC after error@1963817729)))
03:03:33.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.593 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.596 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.633 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.653 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.653 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.653 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.653 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.655 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.673 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.674 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.674 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.674 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.674 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.676 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.754 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [5ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [85ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1787957748(state=PENDING, result=null, callback=(continuation of Deferred@592680685 after retry RPC after error@1302299839) -> (continuation of Deferred@220092466 after retry RPC after error@2012870100) -> (continuation of Deferred@1210249528 after retry RPC after error@2041130751) -> (continuation of Deferred@109651504 after retry RPC after error@1055269984) -> (continuation of Deferred@1637005749 after retry RPC after error@1375514030) -> (continuation of Deferred@504084743 after retry RPC after error@297884546), errback=(continuation of Deferred@592680685 after retry RPC after error@1302299839) -> (continuation of Deferred@220092466 after retry RPC after error@2012870100) -> (continuation of Deferred@1210249528 after retry RPC after error@2041130751) -> (continuation of Deferred@109651504 after retry RPC after error@1055269984) -> (continuation of Deferred@1637005749 after retry RPC after error@1375514030) -> (continuation of Deferred@504084743 after retry RPC after error@297884546)))
03:03:33.755 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [5ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [85ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1787957748(state=PENDING, result=null, callback=(continuation of Deferred@592680685 after retry RPC after error@1302299839) -> (continuation of Deferred@220092466 after retry RPC after error@2012870100) -> (continuation of Deferred@1210249528 after retry RPC after error@2041130751) -> (continuation of Deferred@109651504 after retry RPC after error@1055269984) -> (continuation of Deferred@1637005749 after retry RPC after error@1375514030) -> (continuation of Deferred@504084743 after retry RPC after error@297884546), errback=(continuation of Deferred@592680685 after retry RPC after error@1302299839) -> (continuation of Deferred@220092466 after retry RPC after error@2012870100) -> (continuation of Deferred@1210249528 after retry RPC after error@2041130751) -> (continuation of Deferred@109651504 after retry RPC after error@1055269984) -> (continuation of Deferred@1637005749 after retry RPC after error@1375514030) -> (continuation of Deferred@504084743 after retry RPC after error@297884546)))
03:03:33.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.773 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.780591 17518 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.195s from last keepalive heartbeat (effective timeout is 1.000s)
03:03:33.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:33.781986 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:33.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.813 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.815 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.833 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.833 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.883446 18356 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.883643 18356 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:33.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.884276 18356 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:33.886471 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:33.886663 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.886857 18217 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:33.887179 18356 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:33.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.894 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.895 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:33.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:33.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:33.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:33.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:33.995 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@849251496(state=PENDING, result=null, callback=(continuation of Deferred@446296719 after retry RPC after error@987166540) -> (continuation of Deferred@985399445 after retry RPC after error@2080452774) -> (continuation of Deferred@702134307 after retry RPC after error@616060611) -> (continuation of Deferred@2128237161 after retry RPC after error@187422996) -> (continuation of Deferred@1867997426 after retry RPC after error@155135381) -> (continuation of Deferred@635076545 after retry RPC after error@1886187799) -> (continuation of Deferred@619204376 after retry RPC after error@1414241685), errback=(continuation of Deferred@446296719 after retry RPC after error@987166540) -> (continuation of Deferred@985399445 after retry RPC after error@2080452774) -> (continuation of Deferred@702134307 after retry RPC after error@616060611) -> (continuation of Deferred@2128237161 after retry RPC after error@187422996) -> (continuation of Deferred@1867997426 after retry RPC after error@155135381) -> (continuation of Deferred@635076545 after retry RPC after error@1886187799) -> (continuation of Deferred@619204376 after retry RPC after error@1414241685)))
03:03:33.996 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@849251496(state=PENDING, result=null, callback=(continuation of Deferred@446296719 after retry RPC after error@987166540) -> (continuation of Deferred@985399445 after retry RPC after error@2080452774) -> (continuation of Deferred@702134307 after retry RPC after error@616060611) -> (continuation of Deferred@2128237161 after retry RPC after error@187422996) -> (continuation of Deferred@1867997426 after retry RPC after error@155135381) -> (continuation of Deferred@635076545 after retry RPC after error@1886187799) -> (continuation of Deferred@619204376 after retry RPC after error@1414241685), errback=(continuation of Deferred@446296719 after retry RPC after error@987166540) -> (continuation of Deferred@985399445 after retry RPC after error@2080452774) -> (continuation of Deferred@702134307 after retry RPC after error@616060611) -> (continuation of Deferred@2128237161 after retry RPC after error@187422996) -> (continuation of Deferred@1867997426 after retry RPC after error@155135381) -> (continuation of Deferred@635076545 after retry RPC after error@1886187799) -> (continuation of Deferred@619204376 after retry RPC after error@1414241685)))
03:03:34.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.073 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.235 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.236 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1166235040(state=PENDING, result=null, callback=(continuation of Deferred@1307177805 after retry RPC after error@1772387202) -> (continuation of Deferred@1061982495 after retry RPC after error@1614675203) -> (continuation of Deferred@1657539026 after retry RPC after error@687740514) -> (continuation of Deferred@1944174091 after retry RPC after error@2084278417) -> (continuation of Deferred@2126222429 after retry RPC after error@640598254) -> (continuation of Deferred@1489056710 after retry RPC after error@2019462999) -> (continuation of Deferred@1516631421 after retry RPC after error@169637743), errback=(continuation of Deferred@1307177805 after retry RPC after error@1772387202) -> (continuation of Deferred@1061982495 after retry RPC after error@1614675203) -> (continuation of Deferred@1657539026 after retry RPC after error@687740514) -> (continuation of Deferred@1944174091 after retry RPC after error@2084278417) -> (continuation of Deferred@2126222429 after retry RPC after error@640598254) -> (continuation of Deferred@1489056710 after retry RPC after error@2019462999) -> (continuation of Deferred@1516631421 after retry RPC after error@169637743)))
03:03:34.236 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1166235040(state=PENDING, result=null, callback=(continuation of Deferred@1307177805 after retry RPC after error@1772387202) -> (continuation of Deferred@1061982495 after retry RPC after error@1614675203) -> (continuation of Deferred@1657539026 after retry RPC after error@687740514) -> (continuation of Deferred@1944174091 after retry RPC after error@2084278417) -> (continuation of Deferred@2126222429 after retry RPC after error@640598254) -> (continuation of Deferred@1489056710 after retry RPC after error@2019462999) -> (continuation of Deferred@1516631421 after retry RPC after error@169637743), errback=(continuation of Deferred@1307177805 after retry RPC after error@1772387202) -> (continuation of Deferred@1061982495 after retry RPC after error@1614675203) -> (continuation of Deferred@1657539026 after retry RPC after error@687740514) -> (continuation of Deferred@1944174091 after retry RPC after error@2084278417) -> (continuation of Deferred@2126222429 after retry RPC after error@640598254) -> (continuation of Deferred@1489056710 after retry RPC after error@2019462999) -> (continuation of Deferred@1516631421 after retry RPC after error@169637743)))
03:03:34.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.295 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.313 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.314 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.316 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.353 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.354 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.354 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.355 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.433 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.435 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1118541420(state=PENDING, result=null, callback=(continuation of Deferred@928567462 after retry RPC after error@856844030) -> (continuation of Deferred@319452003 after retry RPC after error@227319704) -> (continuation of Deferred@2082517871 after retry RPC after error@1554325379) -> (continuation of Deferred@477425183 after retry RPC after error@1978516054) -> (continuation of Deferred@212839503 after retry RPC after error@2021847902) -> (continuation of Deferred@1128132167 after retry RPC after error@1482259204) -> (continuation of Deferred@702900628 after retry RPC after error@1184004085), errback=(continuation of Deferred@928567462 after retry RPC after error@856844030) -> (continuation of Deferred@319452003 after retry RPC after error@227319704) -> (continuation of Deferred@2082517871 after retry RPC after error@1554325379) -> (continuation of Deferred@477425183 after retry RPC after error@1978516054) -> (continuation of Deferred@212839503 after retry RPC after error@2021847902) -> (continuation of Deferred@1128132167 after retry RPC after error@1482259204) -> (continuation of Deferred@702900628 after retry RPC after error@1184004085)))
03:03:34.435 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1118541420(state=PENDING, result=null, callback=(continuation of Deferred@928567462 after retry RPC after error@856844030) -> (continuation of Deferred@319452003 after retry RPC after error@227319704) -> (continuation of Deferred@2082517871 after retry RPC after error@1554325379) -> (continuation of Deferred@477425183 after retry RPC after error@1978516054) -> (continuation of Deferred@212839503 after retry RPC after error@2021847902) -> (continuation of Deferred@1128132167 after retry RPC after error@1482259204) -> (continuation of Deferred@702900628 after retry RPC after error@1184004085), errback=(continuation of Deferred@928567462 after retry RPC after error@856844030) -> (continuation of Deferred@319452003 after retry RPC after error@227319704) -> (continuation of Deferred@2082517871 after retry RPC after error@1554325379) -> (continuation of Deferred@477425183 after retry RPC after error@1978516054) -> (continuation of Deferred@212839503 after retry RPC after error@2021847902) -> (continuation of Deferred@1128132167 after retry RPC after error@1482259204) -> (continuation of Deferred@702900628 after retry RPC after error@1184004085)))
03:03:34.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.513 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.513 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.514 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.535 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.594 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.595 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=144), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [144ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1645959247(state=PENDING, result=null, callback=(continuation of Deferred@54840329 after retry RPC after error@1513918192) -> (continuation of Deferred@1998169126 after retry RPC after error@1341674410) -> (continuation of Deferred@1786378968 after retry RPC after error@1418502761) -> (continuation of Deferred@1846412362 after retry RPC after error@1676050210) -> (continuation of Deferred@1181542076 after retry RPC after error@2009242610) -> (continuation of Deferred@1769008404 after retry RPC after error@960462967), errback=(continuation of Deferred@54840329 after retry RPC after error@1513918192) -> (continuation of Deferred@1998169126 after retry RPC after error@1341674410) -> (continuation of Deferred@1786378968 after retry RPC after error@1418502761) -> (continuation of Deferred@1846412362 after retry RPC after error@1676050210) -> (continuation of Deferred@1181542076 after retry RPC after error@2009242610) -> (continuation of Deferred@1769008404 after retry RPC after error@960462967)))
03:03:34.595 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=144), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [144ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1645959247(state=PENDING, result=null, callback=(continuation of Deferred@54840329 after retry RPC after error@1513918192) -> (continuation of Deferred@1998169126 after retry RPC after error@1341674410) -> (continuation of Deferred@1786378968 after retry RPC after error@1418502761) -> (continuation of Deferred@1846412362 after retry RPC after error@1676050210) -> (continuation of Deferred@1181542076 after retry RPC after error@2009242610) -> (continuation of Deferred@1769008404 after retry RPC after error@960462967), errback=(continuation of Deferred@54840329 after retry RPC after error@1513918192) -> (continuation of Deferred@1998169126 after retry RPC after error@1341674410) -> (continuation of Deferred@1786378968 after retry RPC after error@1418502761) -> (continuation of Deferred@1846412362 after retry RPC after error@1676050210) -> (continuation of Deferred@1181542076 after retry RPC after error@2009242610) -> (continuation of Deferred@1769008404 after retry RPC after error@960462967)))
03:03:34.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.615 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:34.672713 18362 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:34.672960 18362 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:34.673682 18362 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:34.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:34.675841 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:34.676115 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:34.676265 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:34.676520 18362 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:34.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.813 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.814 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.871 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1090943250(state=PENDING, result=null, callback=(continuation of Deferred@532756119 after retry RPC after error@240219150) -> (continuation of Deferred@1695842320 after retry RPC after error@396929865) -> (continuation of Deferred@30102181 after retry RPC after error@79669142) -> (continuation of Deferred@1164616465 after retry RPC after error@399161878) -> (continuation of Deferred@1288674352 after retry RPC after error@892666152) -> (continuation of Deferred@626482868 after retry RPC after error@1729055234) -> (continuation of Deferred@1460761807 after retry RPC after error@1253049156) -> (continuation of Deferred@1435321944 after retry RPC after error@1221544369), errback=(continuation of Deferred@532756119 after retry RPC after error@240219150) -> (continuation of Deferred@1695842320 after retry RPC after error@396929865) -> (continuation of Deferred@30102181 after retry RPC after error@79669142) -> (continuation of Deferred@1164616465 after retry RPC after error@399161878) -> (continuation of Deferred@1288674352 after retry RPC after error@892666152) -> (continuation of Deferred@626482868 after retry RPC after error@1729055234) -> (continuation of Deferred@1460761807 after retry RPC after error@1253049156) -> (continuation of Deferred@1435321944 after retry RPC after error@1221544369)))
03:03:34.872 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:34.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.974 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.975 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:34.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:34.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:34.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:34.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:34.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.152 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1760947616(state=PENDING, result=null, callback=(continuation of Deferred@1805678427 after retry RPC after error@1634233778) -> (continuation of Deferred@702418610 after retry RPC after error@1845587652) -> (continuation of Deferred@108250864 after retry RPC after error@617885297) -> (continuation of Deferred@2099650503 after retry RPC after error@414460393) -> (continuation of Deferred@15218761 after retry RPC after error@97134979) -> (continuation of Deferred@1214240474 after retry RPC after error@1906132313) -> (continuation of Deferred@813044051 after retry RPC after error@1688514799), errback=(continuation of Deferred@1805678427 after retry RPC after error@1634233778) -> (continuation of Deferred@702418610 after retry RPC after error@1845587652) -> (continuation of Deferred@108250864 after retry RPC after error@617885297) -> (continuation of Deferred@2099650503 after retry RPC after error@414460393) -> (continuation of Deferred@15218761 after retry RPC after error@97134979) -> (continuation of Deferred@1214240474 after retry RPC after error@1906132313) -> (continuation of Deferred@813044051 after retry RPC after error@1688514799)))
03:03:35.153 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:35.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.253 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.302 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.355 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [132ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [132ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [184ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1083784729(state=PENDING, result=null, callback=(continuation of Deferred@1368076699 after retry RPC after error@381977929) -> (continuation of Deferred@1620468973 after retry RPC after error@1769404176) -> (continuation of Deferred@1635320908 after retry RPC after error@1403337703) -> (continuation of Deferred@13081714 after retry RPC after error@1582769421) -> (continuation of Deferred@1998918127 after retry RPC after error@228943599) -> (continuation of Deferred@834494245 after retry RPC after error@1445296978), errback=(continuation of Deferred@1368076699 after retry RPC after error@381977929) -> (continuation of Deferred@1620468973 after retry RPC after error@1769404176) -> (continuation of Deferred@1635320908 after retry RPC after error@1403337703) -> (continuation of Deferred@13081714 after retry RPC after error@1582769421) -> (continuation of Deferred@1998918127 after retry RPC after error@228943599) -> (continuation of Deferred@834494245 after retry RPC after error@1445296978)))
03:03:35.355 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [132ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [132ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [184ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1083784729(state=PENDING, result=null, callback=(continuation of Deferred@1368076699 after retry RPC after error@381977929) -> (continuation of Deferred@1620468973 after retry RPC after error@1769404176) -> (continuation of Deferred@1635320908 after retry RPC after error@1403337703) -> (continuation of Deferred@13081714 after retry RPC after error@1582769421) -> (continuation of Deferred@1998918127 after retry RPC after error@228943599) -> (continuation of Deferred@834494245 after retry RPC after error@1445296978), errback=(continuation of Deferred@1368076699 after retry RPC after error@381977929) -> (continuation of Deferred@1620468973 after retry RPC after error@1769404176) -> (continuation of Deferred@1635320908 after retry RPC after error@1403337703) -> (continuation of Deferred@13081714 after retry RPC after error@1582769421) -> (continuation of Deferred@1998918127 after retry RPC after error@228943599) -> (continuation of Deferred@834494245 after retry RPC after error@1445296978)))
03:03:35.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.393 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.473 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:35.567273 18370 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:35.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:35.567586 18370 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:35.568290 18370 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:35.571045 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:35.571256 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:35.571439 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:35.571696 18370 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:35.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.613 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.615 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.615 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1284535335(state=PENDING, result=null, callback=(continuation of Deferred@2005157577 after retry RPC after error@1504189843) -> (continuation of Deferred@583616704 after retry RPC after error@1832333810) -> (continuation of Deferred@953192595 after retry RPC after error@554518175) -> (continuation of Deferred@1240106294 after retry RPC after error@486970712) -> (continuation of Deferred@560908899 after retry RPC after error@1780238703) -> (continuation of Deferred@2009248303 after retry RPC after error@1581729242) -> (continuation of Deferred@153985443 after retry RPC after error@32687832), errback=(continuation of Deferred@2005157577 after retry RPC after error@1504189843) -> (continuation of Deferred@583616704 after retry RPC after error@1832333810) -> (continuation of Deferred@953192595 after retry RPC after error@554518175) -> (continuation of Deferred@1240106294 after retry RPC after error@486970712) -> (continuation of Deferred@560908899 after retry RPC after error@1780238703) -> (continuation of Deferred@2009248303 after retry RPC after error@1581729242) -> (continuation of Deferred@153985443 after retry RPC after error@32687832)))
03:03:35.616 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1284535335(state=PENDING, result=null, callback=(continuation of Deferred@2005157577 after retry RPC after error@1504189843) -> (continuation of Deferred@583616704 after retry RPC after error@1832333810) -> (continuation of Deferred@953192595 after retry RPC after error@554518175) -> (continuation of Deferred@1240106294 after retry RPC after error@486970712) -> (continuation of Deferred@560908899 after retry RPC after error@1780238703) -> (continuation of Deferred@2009248303 after retry RPC after error@1581729242) -> (continuation of Deferred@153985443 after retry RPC after error@32687832), errback=(continuation of Deferred@2005157577 after retry RPC after error@1504189843) -> (continuation of Deferred@583616704 after retry RPC after error@1832333810) -> (continuation of Deferred@953192595 after retry RPC after error@554518175) -> (continuation of Deferred@1240106294 after retry RPC after error@486970712) -> (continuation of Deferred@560908899 after retry RPC after error@1780238703) -> (continuation of Deferred@2009248303 after retry RPC after error@1581729242) -> (continuation of Deferred@153985443 after retry RPC after error@32687832)))
03:03:35.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.673 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.753 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.753 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.755 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@801507429(state=PENDING, result=null, callback=(continuation of Deferred@1922841161 after retry RPC after error@1364275334) -> (continuation of Deferred@544332872 after retry RPC after error@1072393475) -> (continuation of Deferred@750143173 after retry RPC after error@1950058197) -> (continuation of Deferred@1518770054 after retry RPC after error@1054964384) -> (continuation of Deferred@545231831 after retry RPC after error@140162083) -> (continuation of Deferred@1720592393 after retry RPC after error@1401911810) -> (continuation of Deferred@263367311 after retry RPC after error@1385495798), errback=(continuation of Deferred@1922841161 after retry RPC after error@1364275334) -> (continuation of Deferred@544332872 after retry RPC after error@1072393475) -> (continuation of Deferred@750143173 after retry RPC after error@1950058197) -> (continuation of Deferred@1518770054 after retry RPC after error@1054964384) -> (continuation of Deferred@545231831 after retry RPC after error@140162083) -> (continuation of Deferred@1720592393 after retry RPC after error@1401911810) -> (continuation of Deferred@263367311 after retry RPC after error@1385495798)))
03:03:35.794 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@801507429(state=PENDING, result=null, callback=(continuation of Deferred@1922841161 after retry RPC after error@1364275334) -> (continuation of Deferred@544332872 after retry RPC after error@1072393475) -> (continuation of Deferred@750143173 after retry RPC after error@1950058197) -> (continuation of Deferred@1518770054 after retry RPC after error@1054964384) -> (continuation of Deferred@545231831 after retry RPC after error@140162083) -> (continuation of Deferred@1720592393 after retry RPC after error@1401911810) -> (continuation of Deferred@263367311 after retry RPC after error@1385495798), errback=(continuation of Deferred@1922841161 after retry RPC after error@1364275334) -> (continuation of Deferred@544332872 after retry RPC after error@1072393475) -> (continuation of Deferred@750143173 after retry RPC after error@1950058197) -> (continuation of Deferred@1518770054 after retry RPC after error@1054964384) -> (continuation of Deferred@545231831 after retry RPC after error@140162083) -> (continuation of Deferred@1720592393 after retry RPC after error@1401911810) -> (continuation of Deferred@263367311 after retry RPC after error@1385495798)))
03:03:35.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:35.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:35.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:35.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:35.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:35.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.054 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@291495006(state=PENDING, result=null, callback=(continuation of Deferred@58361982 after retry RPC after error@2015440470) -> (continuation of Deferred@1492459455 after retry RPC after error@933729747) -> (continuation of Deferred@1623839265 after retry RPC after error@292230128) -> (continuation of Deferred@1628696197 after retry RPC after error@1779562105) -> (continuation of Deferred@1147627968 after retry RPC after error@224007727) -> (continuation of Deferred@1933817543 after retry RPC after error@377758952) -> (continuation of Deferred@1767007889 after retry RPC after error@879034106), errback=(continuation of Deferred@58361982 after retry RPC after error@2015440470) -> (continuation of Deferred@1492459455 after retry RPC after error@933729747) -> (continuation of Deferred@1623839265 after retry RPC after error@292230128) -> (continuation of Deferred@1628696197 after retry RPC after error@1779562105) -> (continuation of Deferred@1147627968 after retry RPC after error@224007727) -> (continuation of Deferred@1933817543 after retry RPC after error@377758952) -> (continuation of Deferred@1767007889 after retry RPC after error@879034106)))
03:03:36.055 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@291495006(state=PENDING, result=null, callback=(continuation of Deferred@58361982 after retry RPC after error@2015440470) -> (continuation of Deferred@1492459455 after retry RPC after error@933729747) -> (continuation of Deferred@1623839265 after retry RPC after error@292230128) -> (continuation of Deferred@1628696197 after retry RPC after error@1779562105) -> (continuation of Deferred@1147627968 after retry RPC after error@224007727) -> (continuation of Deferred@1933817543 after retry RPC after error@377758952) -> (continuation of Deferred@1767007889 after retry RPC after error@879034106), errback=(continuation of Deferred@58361982 after retry RPC after error@2015440470) -> (continuation of Deferred@1492459455 after retry RPC after error@933729747) -> (continuation of Deferred@1623839265 after retry RPC after error@292230128) -> (continuation of Deferred@1628696197 after retry RPC after error@1779562105) -> (continuation of Deferred@1147627968 after retry RPC after error@224007727) -> (continuation of Deferred@1933817543 after retry RPC after error@377758952) -> (continuation of Deferred@1767007889 after retry RPC after error@879034106)))
03:03:36.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.094137 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:03:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.094465 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:36.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.235 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.235332 18378 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.235594 18378 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:36.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.236346 18378 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:36.238252 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:36.238494 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.238622 18217 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:36.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.238875 18378 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:36.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.314 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1113857(state=PENDING, result=null, callback=(continuation of Deferred@426070559 after retry RPC after error@823064512) -> (continuation of Deferred@999895689 after retry RPC after error@1022765580) -> (continuation of Deferred@254829095 after retry RPC after error@1237726454) -> (continuation of Deferred@993614397 after retry RPC after error@915583260) -> (continuation of Deferred@985648307 after retry RPC after error@1109798550) -> (continuation of Deferred@859794525 after retry RPC after error@277225847) -> (continuation of Deferred@1419296128 after retry RPC after error@658078491) -> (continuation of Deferred@1873589797 after retry RPC after error@1317056318), errback=(continuation of Deferred@426070559 after retry RPC after error@823064512) -> (continuation of Deferred@999895689 after retry RPC after error@1022765580) -> (continuation of Deferred@254829095 after retry RPC after error@1237726454) -> (continuation of Deferred@993614397 after retry RPC after error@915583260) -> (continuation of Deferred@985648307 after retry RPC after error@1109798550) -> (continuation of Deferred@859794525 after retry RPC after error@277225847) -> (continuation of Deferred@1419296128 after retry RPC after error@658078491) -> (continuation of Deferred@1873589797 after retry RPC after error@1317056318)))
03:03:36.315 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1113857(state=PENDING, result=null, callback=(continuation of Deferred@426070559 after retry RPC after error@823064512) -> (continuation of Deferred@999895689 after retry RPC after error@1022765580) -> (continuation of Deferred@254829095 after retry RPC after error@1237726454) -> (continuation of Deferred@993614397 after retry RPC after error@915583260) -> (continuation of Deferred@985648307 after retry RPC after error@1109798550) -> (continuation of Deferred@859794525 after retry RPC after error@277225847) -> (continuation of Deferred@1419296128 after retry RPC after error@658078491) -> (continuation of Deferred@1873589797 after retry RPC after error@1317056318), errback=(continuation of Deferred@426070559 after retry RPC after error@823064512) -> (continuation of Deferred@999895689 after retry RPC after error@1022765580) -> (continuation of Deferred@254829095 after retry RPC after error@1237726454) -> (continuation of Deferred@993614397 after retry RPC after error@915583260) -> (continuation of Deferred@985648307 after retry RPC after error@1109798550) -> (continuation of Deferred@859794525 after retry RPC after error@277225847) -> (continuation of Deferred@1419296128 after retry RPC after error@658078491) -> (continuation of Deferred@1873589797 after retry RPC after error@1317056318)))
03:03:36.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.573 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.575 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1789732020(state=PENDING, result=null, callback=(continuation of Deferred@422307675 after retry RPC after error@294523733) -> (continuation of Deferred@517075965 after retry RPC after error@500886551) -> (continuation of Deferred@1536016840 after retry RPC after error@980968791) -> (continuation of Deferred@723986430 after retry RPC after error@1633148121) -> (continuation of Deferred@1424085690 after retry RPC after error@768235436) -> (continuation of Deferred@1061199164 after retry RPC after error@1613639449) -> (continuation of Deferred@973100452 after retry RPC after error@1158156927), errback=(continuation of Deferred@422307675 after retry RPC after error@294523733) -> (continuation of Deferred@517075965 after retry RPC after error@500886551) -> (continuation of Deferred@1536016840 after retry RPC after error@980968791) -> (continuation of Deferred@723986430 after retry RPC after error@1633148121) -> (continuation of Deferred@1424085690 after retry RPC after error@768235436) -> (continuation of Deferred@1061199164 after retry RPC after error@1613639449) -> (continuation of Deferred@973100452 after retry RPC after error@1158156927)))
03:03:36.575 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1789732020(state=PENDING, result=null, callback=(continuation of Deferred@422307675 after retry RPC after error@294523733) -> (continuation of Deferred@517075965 after retry RPC after error@500886551) -> (continuation of Deferred@1536016840 after retry RPC after error@980968791) -> (continuation of Deferred@723986430 after retry RPC after error@1633148121) -> (continuation of Deferred@1424085690 after retry RPC after error@768235436) -> (continuation of Deferred@1061199164 after retry RPC after error@1613639449) -> (continuation of Deferred@973100452 after retry RPC after error@1158156927), errback=(continuation of Deferred@422307675 after retry RPC after error@294523733) -> (continuation of Deferred@517075965 after retry RPC after error@500886551) -> (continuation of Deferred@1536016840 after retry RPC after error@980968791) -> (continuation of Deferred@723986430 after retry RPC after error@1633148121) -> (continuation of Deferred@1424085690 after retry RPC after error@768235436) -> (continuation of Deferred@1061199164 after retry RPC after error@1613639449) -> (continuation of Deferred@973100452 after retry RPC after error@1158156927)))
03:03:36.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.814 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@430770638(state=PENDING, result=null, callback=(continuation of Deferred@1118882901 after retry RPC after error@1475988447) -> (continuation of Deferred@1569446035 after retry RPC after error@748247445) -> (continuation of Deferred@1745873463 after retry RPC after error@1899066330) -> (continuation of Deferred@1929208367 after retry RPC after error@340707348) -> (continuation of Deferred@50845432 after retry RPC after error@2043857633) -> (continuation of Deferred@1621566446 after retry RPC after error@38488454) -> (continuation of Deferred@1061617441 after retry RPC after error@1353498602), errback=(continuation of Deferred@1118882901 after retry RPC after error@1475988447) -> (continuation of Deferred@1569446035 after retry RPC after error@748247445) -> (continuation of Deferred@1745873463 after retry RPC after error@1899066330) -> (continuation of Deferred@1929208367 after retry RPC after error@340707348) -> (continuation of Deferred@50845432 after retry RPC after error@2043857633) -> (continuation of Deferred@1621566446 after retry RPC after error@38488454) -> (continuation of Deferred@1061617441 after retry RPC after error@1353498602)))
03:03:36.815 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@430770638(state=PENDING, result=null, callback=(continuation of Deferred@1118882901 after retry RPC after error@1475988447) -> (continuation of Deferred@1569446035 after retry RPC after error@748247445) -> (continuation of Deferred@1745873463 after retry RPC after error@1899066330) -> (continuation of Deferred@1929208367 after retry RPC after error@340707348) -> (continuation of Deferred@50845432 after retry RPC after error@2043857633) -> (continuation of Deferred@1621566446 after retry RPC after error@38488454) -> (continuation of Deferred@1061617441 after retry RPC after error@1353498602), errback=(continuation of Deferred@1118882901 after retry RPC after error@1475988447) -> (continuation of Deferred@1569446035 after retry RPC after error@748247445) -> (continuation of Deferred@1745873463 after retry RPC after error@1899066330) -> (continuation of Deferred@1929208367 after retry RPC after error@340707348) -> (continuation of Deferred@50845432 after retry RPC after error@2043857633) -> (continuation of Deferred@1621566446 after retry RPC after error@38488454) -> (continuation of Deferred@1061617441 after retry RPC after error@1353498602)))
03:03:36.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.855 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.861617 18386 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:36.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.861842 18386 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:36.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.862636 18386 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:36.863467 18217 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111) [suppressed 12 similar messages]
03:03:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:36.864804 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:36.865064 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.865175 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:36.865420 18386 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:36.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:36.975 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@470031576(state=PENDING, result=null, callback=(continuation of Deferred@1964408599 after retry RPC after error@1058600992) -> (continuation of Deferred@538113014 after retry RPC after error@483187184) -> (continuation of Deferred@1524949945 after retry RPC after error@583387453) -> (continuation of Deferred@360649680 after retry RPC after error@1871076310) -> (continuation of Deferred@302154932 after retry RPC after error@503458792) -> (continuation of Deferred@1938705003 after retry RPC after error@780920615) -> (continuation of Deferred@682291558 after retry RPC after error@662229744), errback=(continuation of Deferred@1964408599 after retry RPC after error@1058600992) -> (continuation of Deferred@538113014 after retry RPC after error@483187184) -> (continuation of Deferred@1524949945 after retry RPC after error@583387453) -> (continuation of Deferred@360649680 after retry RPC after error@1871076310) -> (continuation of Deferred@302154932 after retry RPC after error@503458792) -> (continuation of Deferred@1938705003 after retry RPC after error@780920615) -> (continuation of Deferred@682291558 after retry RPC after error@662229744)))
03:03:36.975 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@470031576(state=PENDING, result=null, callback=(continuation of Deferred@1964408599 after retry RPC after error@1058600992) -> (continuation of Deferred@538113014 after retry RPC after error@483187184) -> (continuation of Deferred@1524949945 after retry RPC after error@583387453) -> (continuation of Deferred@360649680 after retry RPC after error@1871076310) -> (continuation of Deferred@302154932 after retry RPC after error@503458792) -> (continuation of Deferred@1938705003 after retry RPC after error@780920615) -> (continuation of Deferred@682291558 after retry RPC after error@662229744), errback=(continuation of Deferred@1964408599 after retry RPC after error@1058600992) -> (continuation of Deferred@538113014 after retry RPC after error@483187184) -> (continuation of Deferred@1524949945 after retry RPC after error@583387453) -> (continuation of Deferred@360649680 after retry RPC after error@1871076310) -> (continuation of Deferred@302154932 after retry RPC after error@503458792) -> (continuation of Deferred@1938705003 after retry RPC after error@780920615) -> (continuation of Deferred@682291558 after retry RPC after error@662229744)))
03:03:36.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:36.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:36.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:36.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:36.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.073 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.214 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@492237827(state=PENDING, result=null, callback=(continuation of Deferred@465828912 after retry RPC after error@1794937488) -> (continuation of Deferred@362302119 after retry RPC after error@1069830907) -> (continuation of Deferred@1004673871 after retry RPC after error@897873585) -> (continuation of Deferred@1654579156 after retry RPC after error@2135952118) -> (continuation of Deferred@1422085680 after retry RPC after error@1113885671) -> (continuation of Deferred@1452313600 after retry RPC after error@703085470) -> (continuation of Deferred@1729380957 after retry RPC after error@30000875), errback=(continuation of Deferred@465828912 after retry RPC after error@1794937488) -> (continuation of Deferred@362302119 after retry RPC after error@1069830907) -> (continuation of Deferred@1004673871 after retry RPC after error@897873585) -> (continuation of Deferred@1654579156 after retry RPC after error@2135952118) -> (continuation of Deferred@1422085680 after retry RPC after error@1113885671) -> (continuation of Deferred@1452313600 after retry RPC after error@703085470) -> (continuation of Deferred@1729380957 after retry RPC after error@30000875)))
03:03:37.215 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@492237827(state=PENDING, result=null, callback=(continuation of Deferred@465828912 after retry RPC after error@1794937488) -> (continuation of Deferred@362302119 after retry RPC after error@1069830907) -> (continuation of Deferred@1004673871 after retry RPC after error@897873585) -> (continuation of Deferred@1654579156 after retry RPC after error@2135952118) -> (continuation of Deferred@1422085680 after retry RPC after error@1113885671) -> (continuation of Deferred@1452313600 after retry RPC after error@703085470) -> (continuation of Deferred@1729380957 after retry RPC after error@30000875), errback=(continuation of Deferred@465828912 after retry RPC after error@1794937488) -> (continuation of Deferred@362302119 after retry RPC after error@1069830907) -> (continuation of Deferred@1004673871 after retry RPC after error@897873585) -> (continuation of Deferred@1654579156 after retry RPC after error@2135952118) -> (continuation of Deferred@1422085680 after retry RPC after error@1113885671) -> (continuation of Deferred@1452313600 after retry RPC after error@703085470) -> (continuation of Deferred@1729380957 after retry RPC after error@30000875)))
03:03:37.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.253 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.253 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.254 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.254 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.254 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.254 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.254 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.255 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.433 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.435 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.435 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1679329210(state=PENDING, result=null, callback=(continuation of Deferred@1793930556 after retry RPC after error@29397468) -> (continuation of Deferred@1585468694 after retry RPC after error@2114272299) -> (continuation of Deferred@2109907458 after retry RPC after error@2013889047) -> (continuation of Deferred@620576997 after retry RPC after error@1934545973) -> (continuation of Deferred@460630764 after retry RPC after error@794397359) -> (continuation of Deferred@1642929609 after retry RPC after error@1686602429) -> (continuation of Deferred@1511683851 after retry RPC after error@972390933), errback=(continuation of Deferred@1793930556 after retry RPC after error@29397468) -> (continuation of Deferred@1585468694 after retry RPC after error@2114272299) -> (continuation of Deferred@2109907458 after retry RPC after error@2013889047) -> (continuation of Deferred@620576997 after retry RPC after error@1934545973) -> (continuation of Deferred@460630764 after retry RPC after error@794397359) -> (continuation of Deferred@1642929609 after retry RPC after error@1686602429) -> (continuation of Deferred@1511683851 after retry RPC after error@972390933)))
03:03:37.436 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1679329210(state=PENDING, result=null, callback=(continuation of Deferred@1793930556 after retry RPC after error@29397468) -> (continuation of Deferred@1585468694 after retry RPC after error@2114272299) -> (continuation of Deferred@2109907458 after retry RPC after error@2013889047) -> (continuation of Deferred@620576997 after retry RPC after error@1934545973) -> (continuation of Deferred@460630764 after retry RPC after error@794397359) -> (continuation of Deferred@1642929609 after retry RPC after error@1686602429) -> (continuation of Deferred@1511683851 after retry RPC after error@972390933), errback=(continuation of Deferred@1793930556 after retry RPC after error@29397468) -> (continuation of Deferred@1585468694 after retry RPC after error@2114272299) -> (continuation of Deferred@2109907458 after retry RPC after error@2013889047) -> (continuation of Deferred@620576997 after retry RPC after error@1934545973) -> (continuation of Deferred@460630764 after retry RPC after error@794397359) -> (continuation of Deferred@1642929609 after retry RPC after error@1686602429) -> (continuation of Deferred@1511683851 after retry RPC after error@972390933)))
03:03:37.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.493 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.533 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.535 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.654 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1949493410(state=PENDING, result=null, callback=(continuation of Deferred@1952502457 after retry RPC after error@25214768) -> (continuation of Deferred@1381478459 after retry RPC after error@826488107) -> (continuation of Deferred@1128586400 after retry RPC after error@1203339975) -> (continuation of Deferred@679490659 after retry RPC after error@4367925) -> (continuation of Deferred@1694944065 after retry RPC after error@1847775900) -> (continuation of Deferred@1202241559 after retry RPC after error@1380890073) -> (continuation of Deferred@21049362 after retry RPC after error@2031068308), errback=(continuation of Deferred@1952502457 after retry RPC after error@25214768) -> (continuation of Deferred@1381478459 after retry RPC after error@826488107) -> (continuation of Deferred@1128586400 after retry RPC after error@1203339975) -> (continuation of Deferred@679490659 after retry RPC after error@4367925) -> (continuation of Deferred@1694944065 after retry RPC after error@1847775900) -> (continuation of Deferred@1202241559 after retry RPC after error@1380890073) -> (continuation of Deferred@21049362 after retry RPC after error@2031068308)))
03:03:37.655 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1949493410(state=PENDING, result=null, callback=(continuation of Deferred@1952502457 after retry RPC after error@25214768) -> (continuation of Deferred@1381478459 after retry RPC after error@826488107) -> (continuation of Deferred@1128586400 after retry RPC after error@1203339975) -> (continuation of Deferred@679490659 after retry RPC after error@4367925) -> (continuation of Deferred@1694944065 after retry RPC after error@1847775900) -> (continuation of Deferred@1202241559 after retry RPC after error@1380890073) -> (continuation of Deferred@21049362 after retry RPC after error@2031068308), errback=(continuation of Deferred@1952502457 after retry RPC after error@25214768) -> (continuation of Deferred@1381478459 after retry RPC after error@826488107) -> (continuation of Deferred@1128586400 after retry RPC after error@1203339975) -> (continuation of Deferred@679490659 after retry RPC after error@4367925) -> (continuation of Deferred@1694944065 after retry RPC after error@1847775900) -> (continuation of Deferred@1202241559 after retry RPC after error@1380890073) -> (continuation of Deferred@21049362 after retry RPC after error@2031068308)))
03:03:37.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.794 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.794 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.795 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.895 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@748197153(state=PENDING, result=null, callback=(continuation of Deferred@543593459 after retry RPC after error@1575849000) -> (continuation of Deferred@1080179017 after retry RPC after error@683384313) -> (continuation of Deferred@1012300310 after retry RPC after error@225127800) -> (continuation of Deferred@1553135085 after retry RPC after error@682819539) -> (continuation of Deferred@658224665 after retry RPC after error@1936678763) -> (continuation of Deferred@949612918 after retry RPC after error@124440017) -> (continuation of Deferred@2098935899 after retry RPC after error@1954985108), errback=(continuation of Deferred@543593459 after retry RPC after error@1575849000) -> (continuation of Deferred@1080179017 after retry RPC after error@683384313) -> (continuation of Deferred@1012300310 after retry RPC after error@225127800) -> (continuation of Deferred@1553135085 after retry RPC after error@682819539) -> (continuation of Deferred@658224665 after retry RPC after error@1936678763) -> (continuation of Deferred@949612918 after retry RPC after error@124440017) -> (continuation of Deferred@2098935899 after retry RPC after error@1954985108)))
03:03:37.895 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@748197153(state=PENDING, result=null, callback=(continuation of Deferred@543593459 after retry RPC after error@1575849000) -> (continuation of Deferred@1080179017 after retry RPC after error@683384313) -> (continuation of Deferred@1012300310 after retry RPC after error@225127800) -> (continuation of Deferred@1553135085 after retry RPC after error@682819539) -> (continuation of Deferred@658224665 after retry RPC after error@1936678763) -> (continuation of Deferred@949612918 after retry RPC after error@124440017) -> (continuation of Deferred@2098935899 after retry RPC after error@1954985108), errback=(continuation of Deferred@543593459 after retry RPC after error@1575849000) -> (continuation of Deferred@1080179017 after retry RPC after error@683384313) -> (continuation of Deferred@1012300310 after retry RPC after error@225127800) -> (continuation of Deferred@1553135085 after retry RPC after error@682819539) -> (continuation of Deferred@658224665 after retry RPC after error@1936678763) -> (continuation of Deferred@949612918 after retry RPC after error@124440017) -> (continuation of Deferred@2098935899 after retry RPC after error@1954985108)))
03:03:37.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.975 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:37.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:37.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:37.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:37.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:37.995 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.074 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1004999020(state=PENDING, result=null, callback=(continuation of Deferred@1135442876 after retry RPC after error@1315782872) -> (continuation of Deferred@1539865024 after retry RPC after error@1487372196) -> (continuation of Deferred@1887408023 after retry RPC after error@1420847808) -> (continuation of Deferred@1278087151 after retry RPC after error@798426001) -> (continuation of Deferred@588784323 after retry RPC after error@1957276170) -> (continuation of Deferred@1220837895 after retry RPC after error@1599638277), errback=(continuation of Deferred@1135442876 after retry RPC after error@1315782872) -> (continuation of Deferred@1539865024 after retry RPC after error@1487372196) -> (continuation of Deferred@1887408023 after retry RPC after error@1420847808) -> (continuation of Deferred@1278087151 after retry RPC after error@798426001) -> (continuation of Deferred@588784323 after retry RPC after error@1957276170) -> (continuation of Deferred@1220837895 after retry RPC after error@1599638277)))
03:03:38.075 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1004999020(state=PENDING, result=null, callback=(continuation of Deferred@1135442876 after retry RPC after error@1315782872) -> (continuation of Deferred@1539865024 after retry RPC after error@1487372196) -> (continuation of Deferred@1887408023 after retry RPC after error@1420847808) -> (continuation of Deferred@1278087151 after retry RPC after error@798426001) -> (continuation of Deferred@588784323 after retry RPC after error@1957276170) -> (continuation of Deferred@1220837895 after retry RPC after error@1599638277), errback=(continuation of Deferred@1135442876 after retry RPC after error@1315782872) -> (continuation of Deferred@1539865024 after retry RPC after error@1487372196) -> (continuation of Deferred@1887408023 after retry RPC after error@1420847808) -> (continuation of Deferred@1278087151 after retry RPC after error@798426001) -> (continuation of Deferred@588784323 after retry RPC after error@1957276170) -> (continuation of Deferred@1220837895 after retry RPC after error@1599638277)))
03:03:38.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.214 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.214 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.253 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@946152907(state=PENDING, result=null, callback=(continuation of Deferred@590812717 after retry RPC after error@890528263) -> (continuation of Deferred@482814164 after retry RPC after error@228400634) -> (continuation of Deferred@369157146 after retry RPC after error@1226887) -> (continuation of Deferred@632730073 after retry RPC after error@1075907160) -> (continuation of Deferred@1360004722 after retry RPC after error@1146309207) -> (continuation of Deferred@1425736573 after retry RPC after error@473997424), errback=(continuation of Deferred@590812717 after retry RPC after error@890528263) -> (continuation of Deferred@482814164 after retry RPC after error@228400634) -> (continuation of Deferred@369157146 after retry RPC after error@1226887) -> (continuation of Deferred@632730073 after retry RPC after error@1075907160) -> (continuation of Deferred@1360004722 after retry RPC after error@1146309207) -> (continuation of Deferred@1425736573 after retry RPC after error@473997424)))
03:03:38.254 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@946152907(state=PENDING, result=null, callback=(continuation of Deferred@590812717 after retry RPC after error@890528263) -> (continuation of Deferred@482814164 after retry RPC after error@228400634) -> (continuation of Deferred@369157146 after retry RPC after error@1226887) -> (continuation of Deferred@632730073 after retry RPC after error@1075907160) -> (continuation of Deferred@1360004722 after retry RPC after error@1146309207) -> (continuation of Deferred@1425736573 after retry RPC after error@473997424), errback=(continuation of Deferred@590812717 after retry RPC after error@890528263) -> (continuation of Deferred@482814164 after retry RPC after error@228400634) -> (continuation of Deferred@369157146 after retry RPC after error@1226887) -> (continuation of Deferred@632730073 after retry RPC after error@1075907160) -> (continuation of Deferred@1360004722 after retry RPC after error@1146309207) -> (continuation of Deferred@1425736573 after retry RPC after error@473997424)))
03:03:38.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.313 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.333 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.334 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.393 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.393 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.415 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.415 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=144), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1296665362(state=PENDING, result=null, callback=(continuation of Deferred@45253301 after retry RPC after error@826784444) -> (continuation of Deferred@518414567 after retry RPC after error@536242482) -> (continuation of Deferred@164095827 after retry RPC after error@89734797) -> (continuation of Deferred@680892313 after retry RPC after error@75940928) -> (continuation of Deferred@852313852 after retry RPC after error@2116856702) -> (continuation of Deferred@2046935677 after retry RPC after error@1217904057) -> (continuation of Deferred@280361201 after retry RPC after error@190437058), errback=(continuation of Deferred@45253301 after retry RPC after error@826784444) -> (continuation of Deferred@518414567 after retry RPC after error@536242482) -> (continuation of Deferred@164095827 after retry RPC after error@89734797) -> (continuation of Deferred@680892313 after retry RPC after error@75940928) -> (continuation of Deferred@852313852 after retry RPC after error@2116856702) -> (continuation of Deferred@2046935677 after retry RPC after error@1217904057) -> (continuation of Deferred@280361201 after retry RPC after error@190437058)))
03:03:38.416 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=144), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1296665362(state=PENDING, result=null, callback=(continuation of Deferred@45253301 after retry RPC after error@826784444) -> (continuation of Deferred@518414567 after retry RPC after error@536242482) -> (continuation of Deferred@164095827 after retry RPC after error@89734797) -> (continuation of Deferred@680892313 after retry RPC after error@75940928) -> (continuation of Deferred@852313852 after retry RPC after error@2116856702) -> (continuation of Deferred@2046935677 after retry RPC after error@1217904057) -> (continuation of Deferred@280361201 after retry RPC after error@190437058), errback=(continuation of Deferred@45253301 after retry RPC after error@826784444) -> (continuation of Deferred@518414567 after retry RPC after error@536242482) -> (continuation of Deferred@164095827 after retry RPC after error@89734797) -> (continuation of Deferred@680892313 after retry RPC after error@75940928) -> (continuation of Deferred@852313852 after retry RPC after error@2116856702) -> (continuation of Deferred@2046935677 after retry RPC after error@1217904057) -> (continuation of Deferred@280361201 after retry RPC after error@190437058)))
03:03:38.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.534 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.534 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.593 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.654 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1991498193(state=PENDING, result=null, callback=(continuation of Deferred@793436205 after retry RPC after error@1988536742) -> (continuation of Deferred@993733837 after retry RPC after error@1403918122) -> (continuation of Deferred@731766522 after retry RPC after error@1754630582) -> (continuation of Deferred@178949421 after retry RPC after error@698272789) -> (continuation of Deferred@1815199186 after retry RPC after error@682447323) -> (continuation of Deferred@1991275014 after retry RPC after error@689043105) -> (continuation of Deferred@1211044220 after retry RPC after error@172929579), errback=(continuation of Deferred@793436205 after retry RPC after error@1988536742) -> (continuation of Deferred@993733837 after retry RPC after error@1403918122) -> (continuation of Deferred@731766522 after retry RPC after error@1754630582) -> (continuation of Deferred@178949421 after retry RPC after error@698272789) -> (continuation of Deferred@1815199186 after retry RPC after error@682447323) -> (continuation of Deferred@1991275014 after retry RPC after error@689043105) -> (continuation of Deferred@1211044220 after retry RPC after error@172929579)))
03:03:38.655 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1991498193(state=PENDING, result=null, callback=(continuation of Deferred@793436205 after retry RPC after error@1988536742) -> (continuation of Deferred@993733837 after retry RPC after error@1403918122) -> (continuation of Deferred@731766522 after retry RPC after error@1754630582) -> (continuation of Deferred@178949421 after retry RPC after error@698272789) -> (continuation of Deferred@1815199186 after retry RPC after error@682447323) -> (continuation of Deferred@1991275014 after retry RPC after error@689043105) -> (continuation of Deferred@1211044220 after retry RPC after error@172929579), errback=(continuation of Deferred@793436205 after retry RPC after error@1988536742) -> (continuation of Deferred@993733837 after retry RPC after error@1403918122) -> (continuation of Deferred@731766522 after retry RPC after error@1754630582) -> (continuation of Deferred@178949421 after retry RPC after error@698272789) -> (continuation of Deferred@1815199186 after retry RPC after error@682447323) -> (continuation of Deferred@1991275014 after retry RPC after error@689043105) -> (continuation of Deferred@1211044220 after retry RPC after error@172929579)))
03:03:38.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.675 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.793 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:38.832636 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:38.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:38.834143 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:38.835846 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:38.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.894 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1376631826(state=PENDING, result=null, callback=(continuation of Deferred@1404362862 after retry RPC after error@1868870694) -> (continuation of Deferred@1423703080 after retry RPC after error@1795766157) -> (continuation of Deferred@524950802 after retry RPC after error@1430605657) -> (continuation of Deferred@1593099109 after retry RPC after error@2086875753) -> (continuation of Deferred@856556154 after retry RPC after error@1280228549) -> (continuation of Deferred@886682050 after retry RPC after error@1372339530) -> (continuation of Deferred@311617755 after retry RPC after error@721667800), errback=(continuation of Deferred@1404362862 after retry RPC after error@1868870694) -> (continuation of Deferred@1423703080 after retry RPC after error@1795766157) -> (continuation of Deferred@524950802 after retry RPC after error@1430605657) -> (continuation of Deferred@1593099109 after retry RPC after error@2086875753) -> (continuation of Deferred@856556154 after retry RPC after error@1280228549) -> (continuation of Deferred@886682050 after retry RPC after error@1372339530) -> (continuation of Deferred@311617755 after retry RPC after error@721667800)))
03:03:38.895 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1376631826(state=PENDING, result=null, callback=(continuation of Deferred@1404362862 after retry RPC after error@1868870694) -> (continuation of Deferred@1423703080 after retry RPC after error@1795766157) -> (continuation of Deferred@524950802 after retry RPC after error@1430605657) -> (continuation of Deferred@1593099109 after retry RPC after error@2086875753) -> (continuation of Deferred@856556154 after retry RPC after error@1280228549) -> (continuation of Deferred@886682050 after retry RPC after error@1372339530) -> (continuation of Deferred@311617755 after retry RPC after error@721667800), errback=(continuation of Deferred@1404362862 after retry RPC after error@1868870694) -> (continuation of Deferred@1423703080 after retry RPC after error@1795766157) -> (continuation of Deferred@524950802 after retry RPC after error@1430605657) -> (continuation of Deferred@1593099109 after retry RPC after error@2086875753) -> (continuation of Deferred@856556154 after retry RPC after error@1280228549) -> (continuation of Deferred@886682050 after retry RPC after error@1372339530) -> (continuation of Deferred@311617755 after retry RPC after error@721667800)))
03:03:38.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:38.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:38.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:38.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:38.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:38.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.075 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1571140528(state=PENDING, result=null, callback=(continuation of Deferred@947111846 after retry RPC after error@1011427636) -> (continuation of Deferred@1968891804 after retry RPC after error@93516468) -> (continuation of Deferred@595217239 after retry RPC after error@1443878533) -> (continuation of Deferred@1951116767 after retry RPC after error@1725884080) -> (continuation of Deferred@395262415 after retry RPC after error@1067220549) -> (continuation of Deferred@370153788 after retry RPC after error@124425768), errback=(continuation of Deferred@947111846 after retry RPC after error@1011427636) -> (continuation of Deferred@1968891804 after retry RPC after error@93516468) -> (continuation of Deferred@595217239 after retry RPC after error@1443878533) -> (continuation of Deferred@1951116767 after retry RPC after error@1725884080) -> (continuation of Deferred@395262415 after retry RPC after error@1067220549) -> (continuation of Deferred@370153788 after retry RPC after error@124425768)))
03:03:39.076 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1571140528(state=PENDING, result=null, callback=(continuation of Deferred@947111846 after retry RPC after error@1011427636) -> (continuation of Deferred@1968891804 after retry RPC after error@93516468) -> (continuation of Deferred@595217239 after retry RPC after error@1443878533) -> (continuation of Deferred@1951116767 after retry RPC after error@1725884080) -> (continuation of Deferred@395262415 after retry RPC after error@1067220549) -> (continuation of Deferred@370153788 after retry RPC after error@124425768), errback=(continuation of Deferred@947111846 after retry RPC after error@1011427636) -> (continuation of Deferred@1968891804 after retry RPC after error@93516468) -> (continuation of Deferred@595217239 after retry RPC after error@1443878533) -> (continuation of Deferred@1951116767 after retry RPC after error@1725884080) -> (continuation of Deferred@395262415 after retry RPC after error@1067220549) -> (continuation of Deferred@370153788 after retry RPC after error@124425768)))
03:03:39.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.173 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.315 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@392032238(state=PENDING, result=null, callback=(continuation of Deferred@721318967 after retry RPC after error@1369031859) -> (continuation of Deferred@2001874788 after retry RPC after error@252222719) -> (continuation of Deferred@1443761893 after retry RPC after error@1227744459) -> (continuation of Deferred@803396485 after retry RPC after error@1682331106) -> (continuation of Deferred@1108936361 after retry RPC after error@1689565597) -> (continuation of Deferred@1572038781 after retry RPC after error@185393004) -> (continuation of Deferred@27962602 after retry RPC after error@1083409683), errback=(continuation of Deferred@721318967 after retry RPC after error@1369031859) -> (continuation of Deferred@2001874788 after retry RPC after error@252222719) -> (continuation of Deferred@1443761893 after retry RPC after error@1227744459) -> (continuation of Deferred@803396485 after retry RPC after error@1682331106) -> (continuation of Deferred@1108936361 after retry RPC after error@1689565597) -> (continuation of Deferred@1572038781 after retry RPC after error@185393004) -> (continuation of Deferred@27962602 after retry RPC after error@1083409683)))
03:03:39.315 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@392032238(state=PENDING, result=null, callback=(continuation of Deferred@721318967 after retry RPC after error@1369031859) -> (continuation of Deferred@2001874788 after retry RPC after error@252222719) -> (continuation of Deferred@1443761893 after retry RPC after error@1227744459) -> (continuation of Deferred@803396485 after retry RPC after error@1682331106) -> (continuation of Deferred@1108936361 after retry RPC after error@1689565597) -> (continuation of Deferred@1572038781 after retry RPC after error@185393004) -> (continuation of Deferred@27962602 after retry RPC after error@1083409683), errback=(continuation of Deferred@721318967 after retry RPC after error@1369031859) -> (continuation of Deferred@2001874788 after retry RPC after error@252222719) -> (continuation of Deferred@1443761893 after retry RPC after error@1227744459) -> (continuation of Deferred@803396485 after retry RPC after error@1682331106) -> (continuation of Deferred@1108936361 after retry RPC after error@1689565597) -> (continuation of Deferred@1572038781 after retry RPC after error@185393004) -> (continuation of Deferred@27962602 after retry RPC after error@1083409683)))
03:03:39.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.453 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.515 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.555 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.555 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [48ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [49ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1225351855(state=PENDING, result=null, callback=(continuation of Deferred@2014115058 after retry RPC after error@750505477) -> (continuation of Deferred@1991963378 after retry RPC after error@1948580496) -> (continuation of Deferred@1804610434 after retry RPC after error@430512915) -> (continuation of Deferred@941971382 after retry RPC after error@1795607128) -> (continuation of Deferred@27275393 after retry RPC after error@1350615589) -> (continuation of Deferred@1116387172 after retry RPC after error@528740711) -> (continuation of Deferred@531103278 after retry RPC after error@1231066645), errback=(continuation of Deferred@2014115058 after retry RPC after error@750505477) -> (continuation of Deferred@1991963378 after retry RPC after error@1948580496) -> (continuation of Deferred@1804610434 after retry RPC after error@430512915) -> (continuation of Deferred@941971382 after retry RPC after error@1795607128) -> (continuation of Deferred@27275393 after retry RPC after error@1350615589) -> (continuation of Deferred@1116387172 after retry RPC after error@528740711) -> (continuation of Deferred@531103278 after retry RPC after error@1231066645)))
03:03:39.556 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [48ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [49ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1225351855(state=PENDING, result=null, callback=(continuation of Deferred@2014115058 after retry RPC after error@750505477) -> (continuation of Deferred@1991963378 after retry RPC after error@1948580496) -> (continuation of Deferred@1804610434 after retry RPC after error@430512915) -> (continuation of Deferred@941971382 after retry RPC after error@1795607128) -> (continuation of Deferred@27275393 after retry RPC after error@1350615589) -> (continuation of Deferred@1116387172 after retry RPC after error@528740711) -> (continuation of Deferred@531103278 after retry RPC after error@1231066645), errback=(continuation of Deferred@2014115058 after retry RPC after error@750505477) -> (continuation of Deferred@1991963378 after retry RPC after error@1948580496) -> (continuation of Deferred@1804610434 after retry RPC after error@430512915) -> (continuation of Deferred@941971382 after retry RPC after error@1795607128) -> (continuation of Deferred@27275393 after retry RPC after error@1350615589) -> (continuation of Deferred@1116387172 after retry RPC after error@528740711) -> (continuation of Deferred@531103278 after retry RPC after error@1231066645)))
03:03:39.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.573 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.575 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.693 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.693 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:39.745738 18402 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:39.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:39.745911 18402 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:39.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:39.746526 18402 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:39.748889 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:39.749066 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:39.749166 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:39.749385 18402 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:39.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.773 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@222594459(state=PENDING, result=null, callback=(continuation of Deferred@1583517859 after retry RPC after error@1721640910) -> (continuation of Deferred@506706200 after retry RPC after error@930258712) -> (continuation of Deferred@2103437021 after retry RPC after error@1785208559) -> (continuation of Deferred@1771078083 after retry RPC after error@1245268406) -> (continuation of Deferred@1221736226 after retry RPC after error@1659357105) -> (continuation of Deferred@253025463 after retry RPC after error@257966106), errback=(continuation of Deferred@1583517859 after retry RPC after error@1721640910) -> (continuation of Deferred@506706200 after retry RPC after error@930258712) -> (continuation of Deferred@2103437021 after retry RPC after error@1785208559) -> (continuation of Deferred@1771078083 after retry RPC after error@1245268406) -> (continuation of Deferred@1221736226 after retry RPC after error@1659357105) -> (continuation of Deferred@253025463 after retry RPC after error@257966106)))
03:03:39.776 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@222594459(state=PENDING, result=null, callback=(continuation of Deferred@1583517859 after retry RPC after error@1721640910) -> (continuation of Deferred@506706200 after retry RPC after error@930258712) -> (continuation of Deferred@2103437021 after retry RPC after error@1785208559) -> (continuation of Deferred@1771078083 after retry RPC after error@1245268406) -> (continuation of Deferred@1221736226 after retry RPC after error@1659357105) -> (continuation of Deferred@253025463 after retry RPC after error@257966106), errback=(continuation of Deferred@1583517859 after retry RPC after error@1721640910) -> (continuation of Deferred@506706200 after retry RPC after error@930258712) -> (continuation of Deferred@2103437021 after retry RPC after error@1785208559) -> (continuation of Deferred@1771078083 after retry RPC after error@1245268406) -> (continuation of Deferred@1221736226 after retry RPC after error@1659357105) -> (continuation of Deferred@253025463 after retry RPC after error@257966106)))
03:03:39.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.853 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:39.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:39.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:39.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:39.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:39.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.034 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@672792755(state=PENDING, result=null, callback=(continuation of Deferred@1533325418 after retry RPC after error@20322090) -> (continuation of Deferred@112246881 after retry RPC after error@2128037788) -> (continuation of Deferred@13520844 after retry RPC after error@837851064) -> (continuation of Deferred@834923055 after retry RPC after error@1978445868) -> (continuation of Deferred@128852338 after retry RPC after error@844508008) -> (continuation of Deferred@476720543 after retry RPC after error@202867694) -> (continuation of Deferred@2051311279 after retry RPC after error@2043342820), errback=(continuation of Deferred@1533325418 after retry RPC after error@20322090) -> (continuation of Deferred@112246881 after retry RPC after error@2128037788) -> (continuation of Deferred@13520844 after retry RPC after error@837851064) -> (continuation of Deferred@834923055 after retry RPC after error@1978445868) -> (continuation of Deferred@128852338 after retry RPC after error@844508008) -> (continuation of Deferred@476720543 after retry RPC after error@202867694) -> (continuation of Deferred@2051311279 after retry RPC after error@2043342820)))
03:03:40.035 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@672792755(state=PENDING, result=null, callback=(continuation of Deferred@1533325418 after retry RPC after error@20322090) -> (continuation of Deferred@112246881 after retry RPC after error@2128037788) -> (continuation of Deferred@13520844 after retry RPC after error@837851064) -> (continuation of Deferred@834923055 after retry RPC after error@1978445868) -> (continuation of Deferred@128852338 after retry RPC after error@844508008) -> (continuation of Deferred@476720543 after retry RPC after error@202867694) -> (continuation of Deferred@2051311279 after retry RPC after error@2043342820), errback=(continuation of Deferred@1533325418 after retry RPC after error@20322090) -> (continuation of Deferred@112246881 after retry RPC after error@2128037788) -> (continuation of Deferred@13520844 after retry RPC after error@837851064) -> (continuation of Deferred@834923055 after retry RPC after error@1978445868) -> (continuation of Deferred@128852338 after retry RPC after error@844508008) -> (continuation of Deferred@476720543 after retry RPC after error@202867694) -> (continuation of Deferred@2051311279 after retry RPC after error@2043342820)))
03:03:40.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.113 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1040178167(state=PENDING, result=null, callback=(continuation of Deferred@929909682 after retry RPC after error@504225868) -> (continuation of Deferred@1909808529 after retry RPC after error@850792511) -> (continuation of Deferred@1876688128 after retry RPC after error@514162487) -> (continuation of Deferred@1162317991 after retry RPC after error@1207064536) -> (continuation of Deferred@2003931611 after retry RPC after error@1108874437) -> (continuation of Deferred@992175753 after retry RPC after error@1857731234), errback=(continuation of Deferred@929909682 after retry RPC after error@504225868) -> (continuation of Deferred@1909808529 after retry RPC after error@850792511) -> (continuation of Deferred@1876688128 after retry RPC after error@514162487) -> (continuation of Deferred@1162317991 after retry RPC after error@1207064536) -> (continuation of Deferred@2003931611 after retry RPC after error@1108874437) -> (continuation of Deferred@992175753 after retry RPC after error@1857731234)))
03:03:40.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1040178167(state=PENDING, result=null, callback=(continuation of Deferred@929909682 after retry RPC after error@504225868) -> (continuation of Deferred@1909808529 after retry RPC after error@850792511) -> (continuation of Deferred@1876688128 after retry RPC after error@514162487) -> (continuation of Deferred@1162317991 after retry RPC after error@1207064536) -> (continuation of Deferred@2003931611 after retry RPC after error@1108874437) -> (continuation of Deferred@992175753 after retry RPC after error@1857731234), errback=(continuation of Deferred@929909682 after retry RPC after error@504225868) -> (continuation of Deferred@1909808529 after retry RPC after error@850792511) -> (continuation of Deferred@1876688128 after retry RPC after error@514162487) -> (continuation of Deferred@1162317991 after retry RPC after error@1207064536) -> (continuation of Deferred@2003931611 after retry RPC after error@1108874437) -> (continuation of Deferred@992175753 after retry RPC after error@1857731234)))
03:03:40.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.253 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.253 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.353 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@783514197(state=PENDING, result=null, callback=(continuation of Deferred@1320007921 after retry RPC after error@1738544521) -> (continuation of Deferred@261964462 after retry RPC after error@986312922) -> (continuation of Deferred@419474754 after retry RPC after error@2121002934) -> (continuation of Deferred@453397223 after retry RPC after error@1849435773) -> (continuation of Deferred@1925905477 after retry RPC after error@1400744949) -> (continuation of Deferred@2047181327 after retry RPC after error@1971670268), errback=(continuation of Deferred@1320007921 after retry RPC after error@1738544521) -> (continuation of Deferred@261964462 after retry RPC after error@986312922) -> (continuation of Deferred@419474754 after retry RPC after error@2121002934) -> (continuation of Deferred@453397223 after retry RPC after error@1849435773) -> (continuation of Deferred@1925905477 after retry RPC after error@1400744949) -> (continuation of Deferred@2047181327 after retry RPC after error@1971670268)))
03:03:40.354 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@783514197(state=PENDING, result=null, callback=(continuation of Deferred@1320007921 after retry RPC after error@1738544521) -> (continuation of Deferred@261964462 after retry RPC after error@986312922) -> (continuation of Deferred@419474754 after retry RPC after error@2121002934) -> (continuation of Deferred@453397223 after retry RPC after error@1849435773) -> (continuation of Deferred@1925905477 after retry RPC after error@1400744949) -> (continuation of Deferred@2047181327 after retry RPC after error@1971670268), errback=(continuation of Deferred@1320007921 after retry RPC after error@1738544521) -> (continuation of Deferred@261964462 after retry RPC after error@986312922) -> (continuation of Deferred@419474754 after retry RPC after error@2121002934) -> (continuation of Deferred@453397223 after retry RPC after error@1849435773) -> (continuation of Deferred@1925905477 after retry RPC after error@1400744949) -> (continuation of Deferred@2047181327 after retry RPC after error@1971670268)))
03:03:40.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.554 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@140775946(state=PENDING, result=null, callback=(continuation of Deferred@144299631 after retry RPC after error@1911192836) -> (continuation of Deferred@1776083687 after retry RPC after error@1098926484) -> (continuation of Deferred@132957541 after retry RPC after error@634165953) -> (continuation of Deferred@722046789 after retry RPC after error@1753657177) -> (continuation of Deferred@153127105 after retry RPC after error@1216020918) -> (continuation of Deferred@795860059 after retry RPC after error@1926120072), errback=(continuation of Deferred@144299631 after retry RPC after error@1911192836) -> (continuation of Deferred@1776083687 after retry RPC after error@1098926484) -> (continuation of Deferred@132957541 after retry RPC after error@634165953) -> (continuation of Deferred@722046789 after retry RPC after error@1753657177) -> (continuation of Deferred@153127105 after retry RPC after error@1216020918) -> (continuation of Deferred@795860059 after retry RPC after error@1926120072)))
03:03:40.554 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@140775946(state=PENDING, result=null, callback=(continuation of Deferred@144299631 after retry RPC after error@1911192836) -> (continuation of Deferred@1776083687 after retry RPC after error@1098926484) -> (continuation of Deferred@132957541 after retry RPC after error@634165953) -> (continuation of Deferred@722046789 after retry RPC after error@1753657177) -> (continuation of Deferred@153127105 after retry RPC after error@1216020918) -> (continuation of Deferred@795860059 after retry RPC after error@1926120072), errback=(continuation of Deferred@144299631 after retry RPC after error@1911192836) -> (continuation of Deferred@1776083687 after retry RPC after error@1098926484) -> (continuation of Deferred@132957541 after retry RPC after error@634165953) -> (continuation of Deferred@722046789 after retry RPC after error@1753657177) -> (continuation of Deferred@153127105 after retry RPC after error@1216020918) -> (continuation of Deferred@795860059 after retry RPC after error@1926120072)))
03:03:40.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1727065131(state=PENDING, result=null, callback=(continuation of Deferred@147947951 after retry RPC after error@952833643) -> (continuation of Deferred@2130651994 after retry RPC after error@521081114) -> (continuation of Deferred@467085229 after retry RPC after error@2077164274) -> (continuation of Deferred@2087535402 after retry RPC after error@453843580) -> (continuation of Deferred@969260287 after retry RPC after error@983495079) -> (continuation of Deferred@746357197 after retry RPC after error@1607573700) -> (continuation of Deferred@1273482629 after retry RPC after error@112795287), errback=(continuation of Deferred@147947951 after retry RPC after error@952833643) -> (continuation of Deferred@2130651994 after retry RPC after error@521081114) -> (continuation of Deferred@467085229 after retry RPC after error@2077164274) -> (continuation of Deferred@2087535402 after retry RPC after error@453843580) -> (continuation of Deferred@969260287 after retry RPC after error@983495079) -> (continuation of Deferred@746357197 after retry RPC after error@1607573700) -> (continuation of Deferred@1273482629 after retry RPC after error@112795287)))
03:03:40.795 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1727065131(state=PENDING, result=null, callback=(continuation of Deferred@147947951 after retry RPC after error@952833643) -> (continuation of Deferred@2130651994 after retry RPC after error@521081114) -> (continuation of Deferred@467085229 after retry RPC after error@2077164274) -> (continuation of Deferred@2087535402 after retry RPC after error@453843580) -> (continuation of Deferred@969260287 after retry RPC after error@983495079) -> (continuation of Deferred@746357197 after retry RPC after error@1607573700) -> (continuation of Deferred@1273482629 after retry RPC after error@112795287), errback=(continuation of Deferred@147947951 after retry RPC after error@952833643) -> (continuation of Deferred@2130651994 after retry RPC after error@521081114) -> (continuation of Deferred@467085229 after retry RPC after error@2077164274) -> (continuation of Deferred@2087535402 after retry RPC after error@453843580) -> (continuation of Deferred@969260287 after retry RPC after error@983495079) -> (continuation of Deferred@746357197 after retry RPC after error@1607573700) -> (continuation of Deferred@1273482629 after retry RPC after error@112795287)))
03:03:40.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.873 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.873 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.913 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.933 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.935 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:40.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:40.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:40.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:40.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:40.993 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1485934350(state=PENDING, result=null, callback=(continuation of Deferred@954185950 after retry RPC after error@1880415729) -> (continuation of Deferred@1575126243 after retry RPC after error@165648814) -> (continuation of Deferred@1275320579 after retry RPC after error@447643175) -> (continuation of Deferred@1394746381 after retry RPC after error@642110766) -> (continuation of Deferred@1960631076 after retry RPC after error@501927134) -> (continuation of Deferred@1605009698 after retry RPC after error@1464779612) -> (continuation of Deferred@1171863402 after retry RPC after error@236887859) -> (continuation of Deferred@95861827 after retry RPC after error@616815498), errback=(continuation of Deferred@954185950 after retry RPC after error@1880415729) -> (continuation of Deferred@1575126243 after retry RPC after error@165648814) -> (continuation of Deferred@1275320579 after retry RPC after error@447643175) -> (continuation of Deferred@1394746381 after retry RPC after error@642110766) -> (continuation of Deferred@1960631076 after retry RPC after error@501927134) -> (continuation of Deferred@1605009698 after retry RPC after error@1464779612) -> (continuation of Deferred@1171863402 after retry RPC after error@236887859) -> (continuation of Deferred@95861827 after retry RPC after error@616815498)))
03:03:40.994 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1485934350(state=PENDING, result=null, callback=(continuation of Deferred@954185950 after retry RPC after error@1880415729) -> (continuation of Deferred@1575126243 after retry RPC after error@165648814) -> (continuation of Deferred@1275320579 after retry RPC after error@447643175) -> (continuation of Deferred@1394746381 after retry RPC after error@642110766) -> (continuation of Deferred@1960631076 after retry RPC after error@501927134) -> (continuation of Deferred@1605009698 after retry RPC after error@1464779612) -> (continuation of Deferred@1171863402 after retry RPC after error@236887859) -> (continuation of Deferred@95861827 after retry RPC after error@616815498), errback=(continuation of Deferred@954185950 after retry RPC after error@1880415729) -> (continuation of Deferred@1575126243 after retry RPC after error@165648814) -> (continuation of Deferred@1275320579 after retry RPC after error@447643175) -> (continuation of Deferred@1394746381 after retry RPC after error@642110766) -> (continuation of Deferred@1960631076 after retry RPC after error@501927134) -> (continuation of Deferred@1605009698 after retry RPC after error@1464779612) -> (continuation of Deferred@1171863402 after retry RPC after error@236887859) -> (continuation of Deferred@95861827 after retry RPC after error@616815498)))
03:03:41.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:41.095784 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:03:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:41.095963 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:41.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.173 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.174 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.174 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.175 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.272 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1802681879(state=PENDING, result=null, callback=(continuation of Deferred@994990789 after retry RPC after error@1742009388) -> (continuation of Deferred@127336537 after retry RPC after error@2037089466) -> (continuation of Deferred@817520928 after retry RPC after error@1880065679) -> (continuation of Deferred@1327875359 after retry RPC after error@1783551509) -> (continuation of Deferred@152963357 after retry RPC after error@95018319) -> (continuation of Deferred@1690693080 after retry RPC after error@2015846699) -> (continuation of Deferred@17357267 after retry RPC after error@1241530136), errback=(continuation of Deferred@994990789 after retry RPC after error@1742009388) -> (continuation of Deferred@127336537 after retry RPC after error@2037089466) -> (continuation of Deferred@817520928 after retry RPC after error@1880065679) -> (continuation of Deferred@1327875359 after retry RPC after error@1783551509) -> (continuation of Deferred@152963357 after retry RPC after error@95018319) -> (continuation of Deferred@1690693080 after retry RPC after error@2015846699) -> (continuation of Deferred@17357267 after retry RPC after error@1241530136)))
03:03:41.273 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:41.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.313 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.335 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.435 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.453 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.453 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.455 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1304501610(state=PENDING, result=null, callback=(continuation of Deferred@2128551684 after retry RPC after error@161160160) -> (continuation of Deferred@425253614 after retry RPC after error@1547157218) -> (continuation of Deferred@1519824223 after retry RPC after error@184805878) -> (continuation of Deferred@1414778322 after retry RPC after error@896710932) -> (continuation of Deferred@1524700848 after retry RPC after error@1320749123) -> (continuation of Deferred@80634139 after retry RPC after error@617923215) -> (continuation of Deferred@1993479219 after retry RPC after error@1890574778), errback=(continuation of Deferred@2128551684 after retry RPC after error@161160160) -> (continuation of Deferred@425253614 after retry RPC after error@1547157218) -> (continuation of Deferred@1519824223 after retry RPC after error@184805878) -> (continuation of Deferred@1414778322 after retry RPC after error@896710932) -> (continuation of Deferred@1524700848 after retry RPC after error@1320749123) -> (continuation of Deferred@80634139 after retry RPC after error@617923215) -> (continuation of Deferred@1993479219 after retry RPC after error@1890574778)))
03:03:41.456 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1304501610(state=PENDING, result=null, callback=(continuation of Deferred@2128551684 after retry RPC after error@161160160) -> (continuation of Deferred@425253614 after retry RPC after error@1547157218) -> (continuation of Deferred@1519824223 after retry RPC after error@184805878) -> (continuation of Deferred@1414778322 after retry RPC after error@896710932) -> (continuation of Deferred@1524700848 after retry RPC after error@1320749123) -> (continuation of Deferred@80634139 after retry RPC after error@617923215) -> (continuation of Deferred@1993479219 after retry RPC after error@1890574778), errback=(continuation of Deferred@2128551684 after retry RPC after error@161160160) -> (continuation of Deferred@425253614 after retry RPC after error@1547157218) -> (continuation of Deferred@1519824223 after retry RPC after error@184805878) -> (continuation of Deferred@1414778322 after retry RPC after error@896710932) -> (continuation of Deferred@1524700848 after retry RPC after error@1320749123) -> (continuation of Deferred@80634139 after retry RPC after error@617923215) -> (continuation of Deferred@1993479219 after retry RPC after error@1890574778)))
03:03:41.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.513 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.514 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.514 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.515 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.675 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@68259088(state=PENDING, result=null, callback=(continuation of Deferred@1504965029 after retry RPC after error@535752649) -> (continuation of Deferred@618939196 after retry RPC after error@739582909) -> (continuation of Deferred@1793966403 after retry RPC after error@43379568) -> (continuation of Deferred@82752957 after retry RPC after error@23291478) -> (continuation of Deferred@83908832 after retry RPC after error@677637467) -> (continuation of Deferred@1506932407 after retry RPC after error@1134193165), errback=(continuation of Deferred@1504965029 after retry RPC after error@535752649) -> (continuation of Deferred@618939196 after retry RPC after error@739582909) -> (continuation of Deferred@1793966403 after retry RPC after error@43379568) -> (continuation of Deferred@82752957 after retry RPC after error@23291478) -> (continuation of Deferred@83908832 after retry RPC after error@677637467) -> (continuation of Deferred@1506932407 after retry RPC after error@1134193165)))
03:03:41.675 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@68259088(state=PENDING, result=null, callback=(continuation of Deferred@1504965029 after retry RPC after error@535752649) -> (continuation of Deferred@618939196 after retry RPC after error@739582909) -> (continuation of Deferred@1793966403 after retry RPC after error@43379568) -> (continuation of Deferred@82752957 after retry RPC after error@23291478) -> (continuation of Deferred@83908832 after retry RPC after error@677637467) -> (continuation of Deferred@1506932407 after retry RPC after error@1134193165), errback=(continuation of Deferred@1504965029 after retry RPC after error@535752649) -> (continuation of Deferred@618939196 after retry RPC after error@739582909) -> (continuation of Deferred@1793966403 after retry RPC after error@43379568) -> (continuation of Deferred@82752957 after retry RPC after error@23291478) -> (continuation of Deferred@83908832 after retry RPC after error@677637467) -> (continuation of Deferred@1506932407 after retry RPC after error@1134193165)))
03:03:41.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.854 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@612485943(state=PENDING, result=null, callback=(continuation of Deferred@350871039 after retry RPC after error@1302040949) -> (continuation of Deferred@304993389 after retry RPC after error@945782124) -> (continuation of Deferred@1652771261 after retry RPC after error@1929538721) -> (continuation of Deferred@267403280 after retry RPC after error@1275590815) -> (continuation of Deferred@884051459 after retry RPC after error@1412864848) -> (continuation of Deferred@1506517254 after retry RPC after error@721119648), errback=(continuation of Deferred@350871039 after retry RPC after error@1302040949) -> (continuation of Deferred@304993389 after retry RPC after error@945782124) -> (continuation of Deferred@1652771261 after retry RPC after error@1929538721) -> (continuation of Deferred@267403280 after retry RPC after error@1275590815) -> (continuation of Deferred@884051459 after retry RPC after error@1412864848) -> (continuation of Deferred@1506517254 after retry RPC after error@721119648)))
03:03:41.854 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@612485943(state=PENDING, result=null, callback=(continuation of Deferred@350871039 after retry RPC after error@1302040949) -> (continuation of Deferred@304993389 after retry RPC after error@945782124) -> (continuation of Deferred@1652771261 after retry RPC after error@1929538721) -> (continuation of Deferred@267403280 after retry RPC after error@1275590815) -> (continuation of Deferred@884051459 after retry RPC after error@1412864848) -> (continuation of Deferred@1506517254 after retry RPC after error@721119648), errback=(continuation of Deferred@350871039 after retry RPC after error@1302040949) -> (continuation of Deferred@304993389 after retry RPC after error@945782124) -> (continuation of Deferred@1652771261 after retry RPC after error@1929538721) -> (continuation of Deferred@267403280 after retry RPC after error@1275590815) -> (continuation of Deferred@884051459 after retry RPC after error@1412864848) -> (continuation of Deferred@1506517254 after retry RPC after error@721119648)))
03:03:41.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:41.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:41.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:41.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:41.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:41.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.054 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@310008409(state=PENDING, result=null, callback=(continuation of Deferred@1933369400 after retry RPC after error@940457255) -> (continuation of Deferred@718028687 after retry RPC after error@1789136925) -> (continuation of Deferred@2114781265 after retry RPC after error@933494194) -> (continuation of Deferred@2086182030 after retry RPC after error@909529056) -> (continuation of Deferred@548392703 after retry RPC after error@139065653) -> (continuation of Deferred@810395742 after retry RPC after error@880851192) -> (continuation of Deferred@1787187361 after retry RPC after error@759274645), errback=(continuation of Deferred@1933369400 after retry RPC after error@940457255) -> (continuation of Deferred@718028687 after retry RPC after error@1789136925) -> (continuation of Deferred@2114781265 after retry RPC after error@933494194) -> (continuation of Deferred@2086182030 after retry RPC after error@909529056) -> (continuation of Deferred@548392703 after retry RPC after error@139065653) -> (continuation of Deferred@810395742 after retry RPC after error@880851192) -> (continuation of Deferred@1787187361 after retry RPC after error@759274645)))
03:03:42.054 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@310008409(state=PENDING, result=null, callback=(continuation of Deferred@1933369400 after retry RPC after error@940457255) -> (continuation of Deferred@718028687 after retry RPC after error@1789136925) -> (continuation of Deferred@2114781265 after retry RPC after error@933494194) -> (continuation of Deferred@2086182030 after retry RPC after error@909529056) -> (continuation of Deferred@548392703 after retry RPC after error@139065653) -> (continuation of Deferred@810395742 after retry RPC after error@880851192) -> (continuation of Deferred@1787187361 after retry RPC after error@759274645), errback=(continuation of Deferred@1933369400 after retry RPC after error@940457255) -> (continuation of Deferred@718028687 after retry RPC after error@1789136925) -> (continuation of Deferred@2114781265 after retry RPC after error@933494194) -> (continuation of Deferred@2086182030 after retry RPC after error@909529056) -> (continuation of Deferred@548392703 after retry RPC after error@139065653) -> (continuation of Deferred@810395742 after retry RPC after error@880851192) -> (continuation of Deferred@1787187361 after retry RPC after error@759274645)))
03:03:42.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.154 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.154 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.314 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1028793808(state=PENDING, result=null, callback=(continuation of Deferred@1292154476 after retry RPC after error@161440344) -> (continuation of Deferred@681716775 after retry RPC after error@74641478) -> (continuation of Deferred@1814921586 after retry RPC after error@508732173) -> (continuation of Deferred@665348084 after retry RPC after error@877347719) -> (continuation of Deferred@879583853 after retry RPC after error@1059910950) -> (continuation of Deferred@1532998384 after retry RPC after error@904303975) -> (continuation of Deferred@1783101372 after retry RPC after error@840612375), errback=(continuation of Deferred@1292154476 after retry RPC after error@161440344) -> (continuation of Deferred@681716775 after retry RPC after error@74641478) -> (continuation of Deferred@1814921586 after retry RPC after error@508732173) -> (continuation of Deferred@665348084 after retry RPC after error@877347719) -> (continuation of Deferred@879583853 after retry RPC after error@1059910950) -> (continuation of Deferred@1532998384 after retry RPC after error@904303975) -> (continuation of Deferred@1783101372 after retry RPC after error@840612375)))
03:03:42.314 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1028793808(state=PENDING, result=null, callback=(continuation of Deferred@1292154476 after retry RPC after error@161440344) -> (continuation of Deferred@681716775 after retry RPC after error@74641478) -> (continuation of Deferred@1814921586 after retry RPC after error@508732173) -> (continuation of Deferred@665348084 after retry RPC after error@877347719) -> (continuation of Deferred@879583853 after retry RPC after error@1059910950) -> (continuation of Deferred@1532998384 after retry RPC after error@904303975) -> (continuation of Deferred@1783101372 after retry RPC after error@840612375), errback=(continuation of Deferred@1292154476 after retry RPC after error@161440344) -> (continuation of Deferred@681716775 after retry RPC after error@74641478) -> (continuation of Deferred@1814921586 after retry RPC after error@508732173) -> (continuation of Deferred@665348084 after retry RPC after error@877347719) -> (continuation of Deferred@879583853 after retry RPC after error@1059910950) -> (continuation of Deferred@1532998384 after retry RPC after error@904303975) -> (continuation of Deferred@1783101372 after retry RPC after error@840612375)))
03:03:42.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:42.377768 18421 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:42.378072 18421 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:42.378811 18421 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:42.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:42.379856 18217 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:03:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:42.381160 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:42.381328 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:42.381446 18217 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:42.381636 18421 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:42.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.493 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1163486907(state=PENDING, result=null, callback=(continuation of Deferred@2004132443 after retry RPC after error@2132869904) -> (continuation of Deferred@1537178685 after retry RPC after error@1442690379) -> (continuation of Deferred@130659285 after retry RPC after error@1879513535) -> (continuation of Deferred@1348074774 after retry RPC after error@1748389307) -> (continuation of Deferred@581635187 after retry RPC after error@1781085048) -> (continuation of Deferred@1963446886 after retry RPC after error@797315816), errback=(continuation of Deferred@2004132443 after retry RPC after error@2132869904) -> (continuation of Deferred@1537178685 after retry RPC after error@1442690379) -> (continuation of Deferred@130659285 after retry RPC after error@1879513535) -> (continuation of Deferred@1348074774 after retry RPC after error@1748389307) -> (continuation of Deferred@581635187 after retry RPC after error@1781085048) -> (continuation of Deferred@1963446886 after retry RPC after error@797315816)))
03:03:42.494 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1163486907(state=PENDING, result=null, callback=(continuation of Deferred@2004132443 after retry RPC after error@2132869904) -> (continuation of Deferred@1537178685 after retry RPC after error@1442690379) -> (continuation of Deferred@130659285 after retry RPC after error@1879513535) -> (continuation of Deferred@1348074774 after retry RPC after error@1748389307) -> (continuation of Deferred@581635187 after retry RPC after error@1781085048) -> (continuation of Deferred@1963446886 after retry RPC after error@797315816), errback=(continuation of Deferred@2004132443 after retry RPC after error@2132869904) -> (continuation of Deferred@1537178685 after retry RPC after error@1442690379) -> (continuation of Deferred@130659285 after retry RPC after error@1879513535) -> (continuation of Deferred@1348074774 after retry RPC after error@1748389307) -> (continuation of Deferred@581635187 after retry RPC after error@1781085048) -> (continuation of Deferred@1963446886 after retry RPC after error@797315816)))
03:03:42.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.553 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.573 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.593 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.593 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.771 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@773685168(state=PENDING, result=null, callback=(continuation of Deferred@1518725541 after retry RPC after error@1868105145) -> (continuation of Deferred@1220674804 after retry RPC after error@233243299) -> (continuation of Deferred@2019166843 after retry RPC after error@1594473476) -> (continuation of Deferred@120911541 after retry RPC after error@854789866) -> (continuation of Deferred@441631517 after retry RPC after error@145339855) -> (continuation of Deferred@671640369 after retry RPC after error@44479620) -> (continuation of Deferred@241227238 after retry RPC after error@1260523048), errback=(continuation of Deferred@1518725541 after retry RPC after error@1868105145) -> (continuation of Deferred@1220674804 after retry RPC after error@233243299) -> (continuation of Deferred@2019166843 after retry RPC after error@1594473476) -> (continuation of Deferred@120911541 after retry RPC after error@854789866) -> (continuation of Deferred@441631517 after retry RPC after error@145339855) -> (continuation of Deferred@671640369 after retry RPC after error@44479620) -> (continuation of Deferred@241227238 after retry RPC after error@1260523048)))
03:03:42.772 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:42.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:42.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:42.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:42.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:42.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:42.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.052 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@505356521(state=PENDING, result=null, callback=(continuation of Deferred@1360465763 after retry RPC after error@131147556) -> (continuation of Deferred@1760255027 after retry RPC after error@252977583) -> (continuation of Deferred@1712409401 after retry RPC after error@1761065176) -> (continuation of Deferred@1523542833 after retry RPC after error@1962422430) -> (continuation of Deferred@417129915 after retry RPC after error@1303953152) -> (continuation of Deferred@1821365093 after retry RPC after error@1506173610) -> (continuation of Deferred@642295060 after retry RPC after error@771776098), errback=(continuation of Deferred@1360465763 after retry RPC after error@131147556) -> (continuation of Deferred@1760255027 after retry RPC after error@252977583) -> (continuation of Deferred@1712409401 after retry RPC after error@1761065176) -> (continuation of Deferred@1523542833 after retry RPC after error@1962422430) -> (continuation of Deferred@417129915 after retry RPC after error@1303953152) -> (continuation of Deferred@1821365093 after retry RPC after error@1506173610) -> (continuation of Deferred@642295060 after retry RPC after error@771776098)))
03:03:43.052 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:43.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.113 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.313 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.315 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2079835888(state=PENDING, result=null, callback=(continuation of Deferred@1755463504 after retry RPC after error@45485079) -> (continuation of Deferred@103676905 after retry RPC after error@1135111578) -> (continuation of Deferred@973705728 after retry RPC after error@149813218) -> (continuation of Deferred@1643558821 after retry RPC after error@1639109308) -> (continuation of Deferred@392025368 after retry RPC after error@1885889270) -> (continuation of Deferred@611825857 after retry RPC after error@1403496897) -> (continuation of Deferred@715871160 after retry RPC after error@434633673), errback=(continuation of Deferred@1755463504 after retry RPC after error@45485079) -> (continuation of Deferred@103676905 after retry RPC after error@1135111578) -> (continuation of Deferred@973705728 after retry RPC after error@149813218) -> (continuation of Deferred@1643558821 after retry RPC after error@1639109308) -> (continuation of Deferred@392025368 after retry RPC after error@1885889270) -> (continuation of Deferred@611825857 after retry RPC after error@1403496897) -> (continuation of Deferred@715871160 after retry RPC after error@434633673)))
03:03:43.315 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2079835888(state=PENDING, result=null, callback=(continuation of Deferred@1755463504 after retry RPC after error@45485079) -> (continuation of Deferred@103676905 after retry RPC after error@1135111578) -> (continuation of Deferred@973705728 after retry RPC after error@149813218) -> (continuation of Deferred@1643558821 after retry RPC after error@1639109308) -> (continuation of Deferred@392025368 after retry RPC after error@1885889270) -> (continuation of Deferred@611825857 after retry RPC after error@1403496897) -> (continuation of Deferred@715871160 after retry RPC after error@434633673), errback=(continuation of Deferred@1755463504 after retry RPC after error@45485079) -> (continuation of Deferred@103676905 after retry RPC after error@1135111578) -> (continuation of Deferred@973705728 after retry RPC after error@149813218) -> (continuation of Deferred@1643558821 after retry RPC after error@1639109308) -> (continuation of Deferred@392025368 after retry RPC after error@1885889270) -> (continuation of Deferred@611825857 after retry RPC after error@1403496897) -> (continuation of Deferred@715871160 after retry RPC after error@434633673)))
03:03:43.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.574 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@35576606(state=PENDING, result=null, callback=(continuation of Deferred@1079907642 after retry RPC after error@1515077072) -> (continuation of Deferred@1265679139 after retry RPC after error@1061314424) -> (continuation of Deferred@260405871 after retry RPC after error@613316338) -> (continuation of Deferred@1793685731 after retry RPC after error@1207780185) -> (continuation of Deferred@1661935563 after retry RPC after error@878828734) -> (continuation of Deferred@367953073 after retry RPC after error@950332126) -> (continuation of Deferred@600722246 after retry RPC after error@435218349), errback=(continuation of Deferred@1079907642 after retry RPC after error@1515077072) -> (continuation of Deferred@1265679139 after retry RPC after error@1061314424) -> (continuation of Deferred@260405871 after retry RPC after error@613316338) -> (continuation of Deferred@1793685731 after retry RPC after error@1207780185) -> (continuation of Deferred@1661935563 after retry RPC after error@878828734) -> (continuation of Deferred@367953073 after retry RPC after error@950332126) -> (continuation of Deferred@600722246 after retry RPC after error@435218349)))
03:03:43.575 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@35576606(state=PENDING, result=null, callback=(continuation of Deferred@1079907642 after retry RPC after error@1515077072) -> (continuation of Deferred@1265679139 after retry RPC after error@1061314424) -> (continuation of Deferred@260405871 after retry RPC after error@613316338) -> (continuation of Deferred@1793685731 after retry RPC after error@1207780185) -> (continuation of Deferred@1661935563 after retry RPC after error@878828734) -> (continuation of Deferred@367953073 after retry RPC after error@950332126) -> (continuation of Deferred@600722246 after retry RPC after error@435218349), errback=(continuation of Deferred@1079907642 after retry RPC after error@1515077072) -> (continuation of Deferred@1265679139 after retry RPC after error@1061314424) -> (continuation of Deferred@260405871 after retry RPC after error@613316338) -> (continuation of Deferred@1793685731 after retry RPC after error@1207780185) -> (continuation of Deferred@1661935563 after retry RPC after error@878828734) -> (continuation of Deferred@367953073 after retry RPC after error@950332126) -> (continuation of Deferred@600722246 after retry RPC after error@435218349)))
03:03:43.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.593 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.595 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.613 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.614 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.614 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.615 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.635 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.675 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.795 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@476447853(state=PENDING, result=null, callback=(continuation of Deferred@1966677630 after retry RPC after error@1711677272) -> (continuation of Deferred@597263113 after retry RPC after error@267431547) -> (continuation of Deferred@839355157 after retry RPC after error@1689104082) -> (continuation of Deferred@197388421 after retry RPC after error@30018800) -> (continuation of Deferred@261845247 after retry RPC after error@105485536) -> (continuation of Deferred@307942252 after retry RPC after error@1135009883) -> (continuation of Deferred@393112664 after retry RPC after error@1573524673), errback=(continuation of Deferred@1966677630 after retry RPC after error@1711677272) -> (continuation of Deferred@597263113 after retry RPC after error@267431547) -> (continuation of Deferred@839355157 after retry RPC after error@1689104082) -> (continuation of Deferred@197388421 after retry RPC after error@30018800) -> (continuation of Deferred@261845247 after retry RPC after error@105485536) -> (continuation of Deferred@307942252 after retry RPC after error@1135009883) -> (continuation of Deferred@393112664 after retry RPC after error@1573524673)))
03:03:43.795 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@476447853(state=PENDING, result=null, callback=(continuation of Deferred@1966677630 after retry RPC after error@1711677272) -> (continuation of Deferred@597263113 after retry RPC after error@267431547) -> (continuation of Deferred@839355157 after retry RPC after error@1689104082) -> (continuation of Deferred@197388421 after retry RPC after error@30018800) -> (continuation of Deferred@261845247 after retry RPC after error@105485536) -> (continuation of Deferred@307942252 after retry RPC after error@1135009883) -> (continuation of Deferred@393112664 after retry RPC after error@1573524673), errback=(continuation of Deferred@1966677630 after retry RPC after error@1711677272) -> (continuation of Deferred@597263113 after retry RPC after error@267431547) -> (continuation of Deferred@839355157 after retry RPC after error@1689104082) -> (continuation of Deferred@197388421 after retry RPC after error@30018800) -> (continuation of Deferred@261845247 after retry RPC after error@105485536) -> (continuation of Deferred@307942252 after retry RPC after error@1135009883) -> (continuation of Deferred@393112664 after retry RPC after error@1573524673)))
03:03:43.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.815 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.833 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:43.887404 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:43.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:43.889935 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:43.891299 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:43.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:43.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:43.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:43.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:43.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:43.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.054 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1450937240(state=PENDING, result=null, callback=(continuation of Deferred@178504550 after retry RPC after error@1404757483) -> (continuation of Deferred@1318184112 after retry RPC after error@1144816049) -> (continuation of Deferred@1351668387 after retry RPC after error@1409409255) -> (continuation of Deferred@359934643 after retry RPC after error@895815845) -> (continuation of Deferred@1664538976 after retry RPC after error@954927929) -> (continuation of Deferred@209620773 after retry RPC after error@293819959) -> (continuation of Deferred@1152086206 after retry RPC after error@620304849), errback=(continuation of Deferred@178504550 after retry RPC after error@1404757483) -> (continuation of Deferred@1318184112 after retry RPC after error@1144816049) -> (continuation of Deferred@1351668387 after retry RPC after error@1409409255) -> (continuation of Deferred@359934643 after retry RPC after error@895815845) -> (continuation of Deferred@1664538976 after retry RPC after error@954927929) -> (continuation of Deferred@209620773 after retry RPC after error@293819959) -> (continuation of Deferred@1152086206 after retry RPC after error@620304849)))
03:03:44.055 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1450937240(state=PENDING, result=null, callback=(continuation of Deferred@178504550 after retry RPC after error@1404757483) -> (continuation of Deferred@1318184112 after retry RPC after error@1144816049) -> (continuation of Deferred@1351668387 after retry RPC after error@1409409255) -> (continuation of Deferred@359934643 after retry RPC after error@895815845) -> (continuation of Deferred@1664538976 after retry RPC after error@954927929) -> (continuation of Deferred@209620773 after retry RPC after error@293819959) -> (continuation of Deferred@1152086206 after retry RPC after error@620304849), errback=(continuation of Deferred@178504550 after retry RPC after error@1404757483) -> (continuation of Deferred@1318184112 after retry RPC after error@1144816049) -> (continuation of Deferred@1351668387 after retry RPC after error@1409409255) -> (continuation of Deferred@359934643 after retry RPC after error@895815845) -> (continuation of Deferred@1664538976 after retry RPC after error@954927929) -> (continuation of Deferred@209620773 after retry RPC after error@293819959) -> (continuation of Deferred@1152086206 after retry RPC after error@620304849)))
03:03:44.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.113 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.254 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2102697833(state=PENDING, result=null, callback=(continuation of Deferred@1500694796 after retry RPC after error@1611762839) -> (continuation of Deferred@2124979593 after retry RPC after error@668920546) -> (continuation of Deferred@1756065212 after retry RPC after error@78786826) -> (continuation of Deferred@1053601984 after retry RPC after error@1846924289) -> (continuation of Deferred@1566606066 after retry RPC after error@1213190709) -> (continuation of Deferred@333324620 after retry RPC after error@1395350250) -> (continuation of Deferred@168109423 after retry RPC after error@1022201952), errback=(continuation of Deferred@1500694796 after retry RPC after error@1611762839) -> (continuation of Deferred@2124979593 after retry RPC after error@668920546) -> (continuation of Deferred@1756065212 after retry RPC after error@78786826) -> (continuation of Deferred@1053601984 after retry RPC after error@1846924289) -> (continuation of Deferred@1566606066 after retry RPC after error@1213190709) -> (continuation of Deferred@333324620 after retry RPC after error@1395350250) -> (continuation of Deferred@168109423 after retry RPC after error@1022201952)))
03:03:44.255 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2102697833(state=PENDING, result=null, callback=(continuation of Deferred@1500694796 after retry RPC after error@1611762839) -> (continuation of Deferred@2124979593 after retry RPC after error@668920546) -> (continuation of Deferred@1756065212 after retry RPC after error@78786826) -> (continuation of Deferred@1053601984 after retry RPC after error@1846924289) -> (continuation of Deferred@1566606066 after retry RPC after error@1213190709) -> (continuation of Deferred@333324620 after retry RPC after error@1395350250) -> (continuation of Deferred@168109423 after retry RPC after error@1022201952), errback=(continuation of Deferred@1500694796 after retry RPC after error@1611762839) -> (continuation of Deferred@2124979593 after retry RPC after error@668920546) -> (continuation of Deferred@1756065212 after retry RPC after error@78786826) -> (continuation of Deferred@1053601984 after retry RPC after error@1846924289) -> (continuation of Deferred@1566606066 after retry RPC after error@1213190709) -> (continuation of Deferred@333324620 after retry RPC after error@1395350250) -> (continuation of Deferred@168109423 after retry RPC after error@1022201952)))
03:03:44.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.353 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.514 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1679971070(state=PENDING, result=null, callback=(continuation of Deferred@482361221 after retry RPC after error@849213041) -> (continuation of Deferred@2016267058 after retry RPC after error@250528619) -> (continuation of Deferred@332396836 after retry RPC after error@1320991570) -> (continuation of Deferred@1482591724 after retry RPC after error@542369815) -> (continuation of Deferred@1287201760 after retry RPC after error@1744070414) -> (continuation of Deferred@1265219569 after retry RPC after error@1950739931) -> (continuation of Deferred@1902540432 after retry RPC after error@896443301), errback=(continuation of Deferred@482361221 after retry RPC after error@849213041) -> (continuation of Deferred@2016267058 after retry RPC after error@250528619) -> (continuation of Deferred@332396836 after retry RPC after error@1320991570) -> (continuation of Deferred@1482591724 after retry RPC after error@542369815) -> (continuation of Deferred@1287201760 after retry RPC after error@1744070414) -> (continuation of Deferred@1265219569 after retry RPC after error@1950739931) -> (continuation of Deferred@1902540432 after retry RPC after error@896443301)))
03:03:44.514 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1679971070(state=PENDING, result=null, callback=(continuation of Deferred@482361221 after retry RPC after error@849213041) -> (continuation of Deferred@2016267058 after retry RPC after error@250528619) -> (continuation of Deferred@332396836 after retry RPC after error@1320991570) -> (continuation of Deferred@1482591724 after retry RPC after error@542369815) -> (continuation of Deferred@1287201760 after retry RPC after error@1744070414) -> (continuation of Deferred@1265219569 after retry RPC after error@1950739931) -> (continuation of Deferred@1902540432 after retry RPC after error@896443301), errback=(continuation of Deferred@482361221 after retry RPC after error@849213041) -> (continuation of Deferred@2016267058 after retry RPC after error@250528619) -> (continuation of Deferred@332396836 after retry RPC after error@1320991570) -> (continuation of Deferred@1482591724 after retry RPC after error@542369815) -> (continuation of Deferred@1287201760 after retry RPC after error@1744070414) -> (continuation of Deferred@1265219569 after retry RPC after error@1950739931) -> (continuation of Deferred@1902540432 after retry RPC after error@896443301)))
03:03:44.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.774 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1222534187(state=PENDING, result=null, callback=(continuation of Deferred@1831562595 after retry RPC after error@1384541128) -> (continuation of Deferred@971674629 after retry RPC after error@36589615) -> (continuation of Deferred@878208380 after retry RPC after error@1440845844) -> (continuation of Deferred@952527889 after retry RPC after error@1209658382) -> (continuation of Deferred@1067258868 after retry RPC after error@1722111267) -> (continuation of Deferred@1829243650 after retry RPC after error@1794569009) -> (continuation of Deferred@1034112808 after retry RPC after error@1902316018), errback=(continuation of Deferred@1831562595 after retry RPC after error@1384541128) -> (continuation of Deferred@971674629 after retry RPC after error@36589615) -> (continuation of Deferred@878208380 after retry RPC after error@1440845844) -> (continuation of Deferred@952527889 after retry RPC after error@1209658382) -> (continuation of Deferred@1067258868 after retry RPC after error@1722111267) -> (continuation of Deferred@1829243650 after retry RPC after error@1794569009) -> (continuation of Deferred@1034112808 after retry RPC after error@1902316018)))
03:03:44.775 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1222534187(state=PENDING, result=null, callback=(continuation of Deferred@1831562595 after retry RPC after error@1384541128) -> (continuation of Deferred@971674629 after retry RPC after error@36589615) -> (continuation of Deferred@878208380 after retry RPC after error@1440845844) -> (continuation of Deferred@952527889 after retry RPC after error@1209658382) -> (continuation of Deferred@1067258868 after retry RPC after error@1722111267) -> (continuation of Deferred@1829243650 after retry RPC after error@1794569009) -> (continuation of Deferred@1034112808 after retry RPC after error@1902316018), errback=(continuation of Deferred@1831562595 after retry RPC after error@1384541128) -> (continuation of Deferred@971674629 after retry RPC after error@36589615) -> (continuation of Deferred@878208380 after retry RPC after error@1440845844) -> (continuation of Deferred@952527889 after retry RPC after error@1209658382) -> (continuation of Deferred@1067258868 after retry RPC after error@1722111267) -> (continuation of Deferred@1829243650 after retry RPC after error@1794569009) -> (continuation of Deferred@1034112808 after retry RPC after error@1902316018)))
03:03:44.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.913 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.934 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@560195537(state=PENDING, result=null, callback=(continuation of Deferred@1264963054 after retry RPC after error@1789323706) -> (continuation of Deferred@655017878 after retry RPC after error@434762185) -> (continuation of Deferred@2080789927 after retry RPC after error@565217444) -> (continuation of Deferred@1358417267 after retry RPC after error@105257245) -> (continuation of Deferred@1218694943 after retry RPC after error@1844677383) -> (continuation of Deferred@58806175 after retry RPC after error@901324803), errback=(continuation of Deferred@1264963054 after retry RPC after error@1789323706) -> (continuation of Deferred@655017878 after retry RPC after error@434762185) -> (continuation of Deferred@2080789927 after retry RPC after error@565217444) -> (continuation of Deferred@1358417267 after retry RPC after error@105257245) -> (continuation of Deferred@1218694943 after retry RPC after error@1844677383) -> (continuation of Deferred@58806175 after retry RPC after error@901324803)))
03:03:44.935 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@560195537(state=PENDING, result=null, callback=(continuation of Deferred@1264963054 after retry RPC after error@1789323706) -> (continuation of Deferred@655017878 after retry RPC after error@434762185) -> (continuation of Deferred@2080789927 after retry RPC after error@565217444) -> (continuation of Deferred@1358417267 after retry RPC after error@105257245) -> (continuation of Deferred@1218694943 after retry RPC after error@1844677383) -> (continuation of Deferred@58806175 after retry RPC after error@901324803), errback=(continuation of Deferred@1264963054 after retry RPC after error@1789323706) -> (continuation of Deferred@655017878 after retry RPC after error@434762185) -> (continuation of Deferred@2080789927 after retry RPC after error@565217444) -> (continuation of Deferred@1358417267 after retry RPC after error@105257245) -> (continuation of Deferred@1218694943 after retry RPC after error@1844677383) -> (continuation of Deferred@58806175 after retry RPC after error@901324803)))
03:03:44.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:44.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:44.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:44.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:44.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:44.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.033 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.033 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.113 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@970833796(state=PENDING, result=null, callback=(continuation of Deferred@686202580 after retry RPC after error@1704059728) -> (continuation of Deferred@430634307 after retry RPC after error@1306478701) -> (continuation of Deferred@1364422460 after retry RPC after error@993172773) -> (continuation of Deferred@1971091437 after retry RPC after error@1113048335) -> (continuation of Deferred@137853174 after retry RPC after error@658704224) -> (continuation of Deferred@823582586 after retry RPC after error@1293224938) -> (continuation of Deferred@2127964790 after retry RPC after error@1100040907), errback=(continuation of Deferred@686202580 after retry RPC after error@1704059728) -> (continuation of Deferred@430634307 after retry RPC after error@1306478701) -> (continuation of Deferred@1364422460 after retry RPC after error@993172773) -> (continuation of Deferred@1971091437 after retry RPC after error@1113048335) -> (continuation of Deferred@137853174 after retry RPC after error@658704224) -> (continuation of Deferred@823582586 after retry RPC after error@1293224938) -> (continuation of Deferred@2127964790 after retry RPC after error@1100040907)))
03:03:45.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@970833796(state=PENDING, result=null, callback=(continuation of Deferred@686202580 after retry RPC after error@1704059728) -> (continuation of Deferred@430634307 after retry RPC after error@1306478701) -> (continuation of Deferred@1364422460 after retry RPC after error@993172773) -> (continuation of Deferred@1971091437 after retry RPC after error@1113048335) -> (continuation of Deferred@137853174 after retry RPC after error@658704224) -> (continuation of Deferred@823582586 after retry RPC after error@1293224938) -> (continuation of Deferred@2127964790 after retry RPC after error@1100040907), errback=(continuation of Deferred@686202580 after retry RPC after error@1704059728) -> (continuation of Deferred@430634307 after retry RPC after error@1306478701) -> (continuation of Deferred@1364422460 after retry RPC after error@993172773) -> (continuation of Deferred@1971091437 after retry RPC after error@1113048335) -> (continuation of Deferred@137853174 after retry RPC after error@658704224) -> (continuation of Deferred@823582586 after retry RPC after error@1293224938) -> (continuation of Deferred@2127964790 after retry RPC after error@1100040907)))
03:03:45.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.418 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1944405152(state=PENDING, result=null, callback=(continuation of Deferred@430130076 after retry RPC after error@652637679) -> (continuation of Deferred@704131515 after retry RPC after error@1893838944) -> (continuation of Deferred@1935520323 after retry RPC after error@1719789770) -> (continuation of Deferred@236746824 after retry RPC after error@1938013019) -> (continuation of Deferred@1834143938 after retry RPC after error@83665434) -> (continuation of Deferred@1766802677 after retry RPC after error@125334500), errback=(continuation of Deferred@430130076 after retry RPC after error@652637679) -> (continuation of Deferred@704131515 after retry RPC after error@1893838944) -> (continuation of Deferred@1935520323 after retry RPC after error@1719789770) -> (continuation of Deferred@236746824 after retry RPC after error@1938013019) -> (continuation of Deferred@1834143938 after retry RPC after error@83665434) -> (continuation of Deferred@1766802677 after retry RPC after error@125334500)))
03:03:45.418 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1944405152(state=PENDING, result=null, callback=(continuation of Deferred@430130076 after retry RPC after error@652637679) -> (continuation of Deferred@704131515 after retry RPC after error@1893838944) -> (continuation of Deferred@1935520323 after retry RPC after error@1719789770) -> (continuation of Deferred@236746824 after retry RPC after error@1938013019) -> (continuation of Deferred@1834143938 after retry RPC after error@83665434) -> (continuation of Deferred@1766802677 after retry RPC after error@125334500), errback=(continuation of Deferred@430130076 after retry RPC after error@652637679) -> (continuation of Deferred@704131515 after retry RPC after error@1893838944) -> (continuation of Deferred@1935520323 after retry RPC after error@1719789770) -> (continuation of Deferred@236746824 after retry RPC after error@1938013019) -> (continuation of Deferred@1834143938 after retry RPC after error@83665434) -> (continuation of Deferred@1766802677 after retry RPC after error@125334500)))
03:03:45.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.654 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1283217052(state=PENDING, result=null, callback=(continuation of Deferred@76553336 after retry RPC after error@585565356) -> (continuation of Deferred@495377142 after retry RPC after error@861972332) -> (continuation of Deferred@924257956 after retry RPC after error@760196750) -> (continuation of Deferred@104169659 after retry RPC after error@811076822) -> (continuation of Deferred@1068848727 after retry RPC after error@1868477681) -> (continuation of Deferred@1162134009 after retry RPC after error@889871163) -> (continuation of Deferred@546874403 after retry RPC after error@981165565), errback=(continuation of Deferred@76553336 after retry RPC after error@585565356) -> (continuation of Deferred@495377142 after retry RPC after error@861972332) -> (continuation of Deferred@924257956 after retry RPC after error@760196750) -> (continuation of Deferred@104169659 after retry RPC after error@811076822) -> (continuation of Deferred@1068848727 after retry RPC after error@1868477681) -> (continuation of Deferred@1162134009 after retry RPC after error@889871163) -> (continuation of Deferred@546874403 after retry RPC after error@981165565)))
03:03:45.654 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1283217052(state=PENDING, result=null, callback=(continuation of Deferred@76553336 after retry RPC after error@585565356) -> (continuation of Deferred@495377142 after retry RPC after error@861972332) -> (continuation of Deferred@924257956 after retry RPC after error@760196750) -> (continuation of Deferred@104169659 after retry RPC after error@811076822) -> (continuation of Deferred@1068848727 after retry RPC after error@1868477681) -> (continuation of Deferred@1162134009 after retry RPC after error@889871163) -> (continuation of Deferred@546874403 after retry RPC after error@981165565), errback=(continuation of Deferred@76553336 after retry RPC after error@585565356) -> (continuation of Deferred@495377142 after retry RPC after error@861972332) -> (continuation of Deferred@924257956 after retry RPC after error@760196750) -> (continuation of Deferred@104169659 after retry RPC after error@811076822) -> (continuation of Deferred@1068848727 after retry RPC after error@1868477681) -> (continuation of Deferred@1162134009 after retry RPC after error@889871163) -> (continuation of Deferred@546874403 after retry RPC after error@981165565)))
03:03:45.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.931 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@830108277(state=PENDING, result=null, callback=(continuation of Deferred@1246704658 after retry RPC after error@61229458) -> (continuation of Deferred@642592322 after retry RPC after error@1153833252) -> (continuation of Deferred@2006604298 after retry RPC after error@1178695602) -> (continuation of Deferred@154234374 after retry RPC after error@2102020413) -> (continuation of Deferred@1478825620 after retry RPC after error@353638085) -> (continuation of Deferred@391580479 after retry RPC after error@956017481) -> (continuation of Deferred@1218091892 after retry RPC after error@2106396316) -> (continuation of Deferred@1378675937 after retry RPC after error@937993230), errback=(continuation of Deferred@1246704658 after retry RPC after error@61229458) -> (continuation of Deferred@642592322 after retry RPC after error@1153833252) -> (continuation of Deferred@2006604298 after retry RPC after error@1178695602) -> (continuation of Deferred@154234374 after retry RPC after error@2102020413) -> (continuation of Deferred@1478825620 after retry RPC after error@353638085) -> (continuation of Deferred@391580479 after retry RPC after error@956017481) -> (continuation of Deferred@1218091892 after retry RPC after error@2106396316) -> (continuation of Deferred@1378675937 after retry RPC after error@937993230)))
03:03:45.932 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:45.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:45.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:45.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:45.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:45.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:45.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.015 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.032 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:46.097371 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:03:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:46.097673 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:46.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1347406958(state=PENDING, result=null, callback=(continuation of Deferred@788942477 after retry RPC after error@303930408) -> (continuation of Deferred@729087039 after retry RPC after error@100039043) -> (continuation of Deferred@412691862 after retry RPC after error@1721437809) -> (continuation of Deferred@1778549657 after retry RPC after error@1552860475) -> (continuation of Deferred@229287585 after retry RPC after error@863820259) -> (continuation of Deferred@1302108766 after retry RPC after error@1729178059) -> (continuation of Deferred@1062605862 after retry RPC after error@1789948398) -> (continuation of Deferred@1129793676 after retry RPC after error@770076252), errback=(continuation of Deferred@788942477 after retry RPC after error@303930408) -> (continuation of Deferred@729087039 after retry RPC after error@100039043) -> (continuation of Deferred@412691862 after retry RPC after error@1721437809) -> (continuation of Deferred@1778549657 after retry RPC after error@1552860475) -> (continuation of Deferred@229287585 after retry RPC after error@863820259) -> (continuation of Deferred@1302108766 after retry RPC after error@1729178059) -> (continuation of Deferred@1062605862 after retry RPC after error@1789948398) -> (continuation of Deferred@1129793676 after retry RPC after error@770076252)))
03:03:46.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1347406958(state=PENDING, result=null, callback=(continuation of Deferred@788942477 after retry RPC after error@303930408) -> (continuation of Deferred@729087039 after retry RPC after error@100039043) -> (continuation of Deferred@412691862 after retry RPC after error@1721437809) -> (continuation of Deferred@1778549657 after retry RPC after error@1552860475) -> (continuation of Deferred@229287585 after retry RPC after error@863820259) -> (continuation of Deferred@1302108766 after retry RPC after error@1729178059) -> (continuation of Deferred@1062605862 after retry RPC after error@1789948398) -> (continuation of Deferred@1129793676 after retry RPC after error@770076252), errback=(continuation of Deferred@788942477 after retry RPC after error@303930408) -> (continuation of Deferred@729087039 after retry RPC after error@100039043) -> (continuation of Deferred@412691862 after retry RPC after error@1721437809) -> (continuation of Deferred@1778549657 after retry RPC after error@1552860475) -> (continuation of Deferred@229287585 after retry RPC after error@863820259) -> (continuation of Deferred@1302108766 after retry RPC after error@1729178059) -> (continuation of Deferred@1062605862 after retry RPC after error@1789948398) -> (continuation of Deferred@1129793676 after retry RPC after error@770076252)))
03:03:46.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.434 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@757549220(state=PENDING, result=null, callback=(continuation of Deferred@778849 after retry RPC after error@639516280) -> (continuation of Deferred@1696129815 after retry RPC after error@1928197496) -> (continuation of Deferred@757729710 after retry RPC after error@2056261517) -> (continuation of Deferred@1488079763 after retry RPC after error@1172906952) -> (continuation of Deferred@947070370 after retry RPC after error@1122347334) -> (continuation of Deferred@138517594 after retry RPC after error@70912505) -> (continuation of Deferred@633697001 after retry RPC after error@1241858190), errback=(continuation of Deferred@778849 after retry RPC after error@639516280) -> (continuation of Deferred@1696129815 after retry RPC after error@1928197496) -> (continuation of Deferred@757729710 after retry RPC after error@2056261517) -> (continuation of Deferred@1488079763 after retry RPC after error@1172906952) -> (continuation of Deferred@947070370 after retry RPC after error@1122347334) -> (continuation of Deferred@138517594 after retry RPC after error@70912505) -> (continuation of Deferred@633697001 after retry RPC after error@1241858190)))
03:03:46.435 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@757549220(state=PENDING, result=null, callback=(continuation of Deferred@778849 after retry RPC after error@639516280) -> (continuation of Deferred@1696129815 after retry RPC after error@1928197496) -> (continuation of Deferred@757729710 after retry RPC after error@2056261517) -> (continuation of Deferred@1488079763 after retry RPC after error@1172906952) -> (continuation of Deferred@947070370 after retry RPC after error@1122347334) -> (continuation of Deferred@138517594 after retry RPC after error@70912505) -> (continuation of Deferred@633697001 after retry RPC after error@1241858190), errback=(continuation of Deferred@778849 after retry RPC after error@639516280) -> (continuation of Deferred@1696129815 after retry RPC after error@1928197496) -> (continuation of Deferred@757729710 after retry RPC after error@2056261517) -> (continuation of Deferred@1488079763 after retry RPC after error@1172906952) -> (continuation of Deferred@947070370 after retry RPC after error@1122347334) -> (continuation of Deferred@138517594 after retry RPC after error@70912505) -> (continuation of Deferred@633697001 after retry RPC after error@1241858190)))
03:03:46.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.712 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@630752732(state=PENDING, result=null, callback=(continuation of Deferred@1117987589 after retry RPC after error@1616666782) -> (continuation of Deferred@1766042060 after retry RPC after error@135554909) -> (continuation of Deferred@1408554577 after retry RPC after error@1353651233) -> (continuation of Deferred@609671729 after retry RPC after error@265514857) -> (continuation of Deferred@2053142172 after retry RPC after error@1380350957) -> (continuation of Deferred@1087616198 after retry RPC after error@1424958354) -> (continuation of Deferred@767716101 after retry RPC after error@1287320643) -> (continuation of Deferred@294784150 after retry RPC after error@1011633797), errback=(continuation of Deferred@1117987589 after retry RPC after error@1616666782) -> (continuation of Deferred@1766042060 after retry RPC after error@135554909) -> (continuation of Deferred@1408554577 after retry RPC after error@1353651233) -> (continuation of Deferred@609671729 after retry RPC after error@265514857) -> (continuation of Deferred@2053142172 after retry RPC after error@1380350957) -> (continuation of Deferred@1087616198 after retry RPC after error@1424958354) -> (continuation of Deferred@767716101 after retry RPC after error@1287320643) -> (continuation of Deferred@294784150 after retry RPC after error@1011633797)))
03:03:46.713 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:46.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.953 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1432521390(state=PENDING, result=null, callback=(continuation of Deferred@1871910064 after retry RPC after error@372187277) -> (continuation of Deferred@749342715 after retry RPC after error@1843189562) -> (continuation of Deferred@438134949 after retry RPC after error@1968769059) -> (continuation of Deferred@158336164 after retry RPC after error@35551159) -> (continuation of Deferred@1941405457 after retry RPC after error@1309833556) -> (continuation of Deferred@1196298518 after retry RPC after error@916219271) -> (continuation of Deferred@2115750834 after retry RPC after error@597607932), errback=(continuation of Deferred@1871910064 after retry RPC after error@372187277) -> (continuation of Deferred@749342715 after retry RPC after error@1843189562) -> (continuation of Deferred@438134949 after retry RPC after error@1968769059) -> (continuation of Deferred@158336164 after retry RPC after error@35551159) -> (continuation of Deferred@1941405457 after retry RPC after error@1309833556) -> (continuation of Deferred@1196298518 after retry RPC after error@916219271) -> (continuation of Deferred@2115750834 after retry RPC after error@597607932)))
03:03:46.954 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1432521390(state=PENDING, result=null, callback=(continuation of Deferred@1871910064 after retry RPC after error@372187277) -> (continuation of Deferred@749342715 after retry RPC after error@1843189562) -> (continuation of Deferred@438134949 after retry RPC after error@1968769059) -> (continuation of Deferred@158336164 after retry RPC after error@35551159) -> (continuation of Deferred@1941405457 after retry RPC after error@1309833556) -> (continuation of Deferred@1196298518 after retry RPC after error@916219271) -> (continuation of Deferred@2115750834 after retry RPC after error@597607932), errback=(continuation of Deferred@1871910064 after retry RPC after error@372187277) -> (continuation of Deferred@749342715 after retry RPC after error@1843189562) -> (continuation of Deferred@438134949 after retry RPC after error@1968769059) -> (continuation of Deferred@158336164 after retry RPC after error@35551159) -> (continuation of Deferred@1941405457 after retry RPC after error@1309833556) -> (continuation of Deferred@1196298518 after retry RPC after error@916219271) -> (continuation of Deferred@2115750834 after retry RPC after error@597607932)))
03:03:46.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:46.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:46.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:46.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:46.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:46.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1306782199(state=PENDING, result=null, callback=(continuation of Deferred@310780003 after retry RPC after error@1072880395) -> (continuation of Deferred@750626447 after retry RPC after error@2126728894) -> (continuation of Deferred@1575353405 after retry RPC after error@1961250313) -> (continuation of Deferred@892209092 after retry RPC after error@1375476065) -> (continuation of Deferred@587862903 after retry RPC after error@1700431641) -> (continuation of Deferred@546286673 after retry RPC after error@294227504) -> (continuation of Deferred@1651226333 after retry RPC after error@1396217798), errback=(continuation of Deferred@310780003 after retry RPC after error@1072880395) -> (continuation of Deferred@750626447 after retry RPC after error@2126728894) -> (continuation of Deferred@1575353405 after retry RPC after error@1961250313) -> (continuation of Deferred@892209092 after retry RPC after error@1375476065) -> (continuation of Deferred@587862903 after retry RPC after error@1700431641) -> (continuation of Deferred@546286673 after retry RPC after error@294227504) -> (continuation of Deferred@1651226333 after retry RPC after error@1396217798)))
03:03:47.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1306782199(state=PENDING, result=null, callback=(continuation of Deferred@310780003 after retry RPC after error@1072880395) -> (continuation of Deferred@750626447 after retry RPC after error@2126728894) -> (continuation of Deferred@1575353405 after retry RPC after error@1961250313) -> (continuation of Deferred@892209092 after retry RPC after error@1375476065) -> (continuation of Deferred@587862903 after retry RPC after error@1700431641) -> (continuation of Deferred@546286673 after retry RPC after error@294227504) -> (continuation of Deferred@1651226333 after retry RPC after error@1396217798), errback=(continuation of Deferred@310780003 after retry RPC after error@1072880395) -> (continuation of Deferred@750626447 after retry RPC after error@2126728894) -> (continuation of Deferred@1575353405 after retry RPC after error@1961250313) -> (continuation of Deferred@892209092 after retry RPC after error@1375476065) -> (continuation of Deferred@587862903 after retry RPC after error@1700431641) -> (continuation of Deferred@546286673 after retry RPC after error@294227504) -> (continuation of Deferred@1651226333 after retry RPC after error@1396217798)))
03:03:47.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.414 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1163840728(state=PENDING, result=null, callback=(continuation of Deferred@1578319598 after retry RPC after error@2034356245) -> (continuation of Deferred@1494668009 after retry RPC after error@1788738847) -> (continuation of Deferred@759061228 after retry RPC after error@1510532646) -> (continuation of Deferred@904324628 after retry RPC after error@806781106) -> (continuation of Deferred@1864389990 after retry RPC after error@1921296242) -> (continuation of Deferred@1892694384 after retry RPC after error@1950819528) -> (continuation of Deferred@421643331 after retry RPC after error@1321819938), errback=(continuation of Deferred@1578319598 after retry RPC after error@2034356245) -> (continuation of Deferred@1494668009 after retry RPC after error@1788738847) -> (continuation of Deferred@759061228 after retry RPC after error@1510532646) -> (continuation of Deferred@904324628 after retry RPC after error@806781106) -> (continuation of Deferred@1864389990 after retry RPC after error@1921296242) -> (continuation of Deferred@1892694384 after retry RPC after error@1950819528) -> (continuation of Deferred@421643331 after retry RPC after error@1321819938)))
03:03:47.414 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1163840728(state=PENDING, result=null, callback=(continuation of Deferred@1578319598 after retry RPC after error@2034356245) -> (continuation of Deferred@1494668009 after retry RPC after error@1788738847) -> (continuation of Deferred@759061228 after retry RPC after error@1510532646) -> (continuation of Deferred@904324628 after retry RPC after error@806781106) -> (continuation of Deferred@1864389990 after retry RPC after error@1921296242) -> (continuation of Deferred@1892694384 after retry RPC after error@1950819528) -> (continuation of Deferred@421643331 after retry RPC after error@1321819938), errback=(continuation of Deferred@1578319598 after retry RPC after error@2034356245) -> (continuation of Deferred@1494668009 after retry RPC after error@1788738847) -> (continuation of Deferred@759061228 after retry RPC after error@1510532646) -> (continuation of Deferred@904324628 after retry RPC after error@806781106) -> (continuation of Deferred@1864389990 after retry RPC after error@1921296242) -> (continuation of Deferred@1892694384 after retry RPC after error@1950819528) -> (continuation of Deferred@421643331 after retry RPC after error@1321819938)))
03:03:47.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.614 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@461924633(state=PENDING, result=null, callback=(continuation of Deferred@1327015672 after retry RPC after error@2072956523) -> (continuation of Deferred@1176736526 after retry RPC after error@507247218) -> (continuation of Deferred@337551403 after retry RPC after error@435083605) -> (continuation of Deferred@1125094021 after retry RPC after error@44435708) -> (continuation of Deferred@1675165563 after retry RPC after error@202618404) -> (continuation of Deferred@859281049 after retry RPC after error@23366123), errback=(continuation of Deferred@1327015672 after retry RPC after error@2072956523) -> (continuation of Deferred@1176736526 after retry RPC after error@507247218) -> (continuation of Deferred@337551403 after retry RPC after error@435083605) -> (continuation of Deferred@1125094021 after retry RPC after error@44435708) -> (continuation of Deferred@1675165563 after retry RPC after error@202618404) -> (continuation of Deferred@859281049 after retry RPC after error@23366123)))
03:03:47.615 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@461924633(state=PENDING, result=null, callback=(continuation of Deferred@1327015672 after retry RPC after error@2072956523) -> (continuation of Deferred@1176736526 after retry RPC after error@507247218) -> (continuation of Deferred@337551403 after retry RPC after error@435083605) -> (continuation of Deferred@1125094021 after retry RPC after error@44435708) -> (continuation of Deferred@1675165563 after retry RPC after error@202618404) -> (continuation of Deferred@859281049 after retry RPC after error@23366123), errback=(continuation of Deferred@1327015672 after retry RPC after error@2072956523) -> (continuation of Deferred@1176736526 after retry RPC after error@507247218) -> (continuation of Deferred@337551403 after retry RPC after error@435083605) -> (continuation of Deferred@1125094021 after retry RPC after error@44435708) -> (continuation of Deferred@1675165563 after retry RPC after error@202618404) -> (continuation of Deferred@859281049 after retry RPC after error@23366123)))
03:03:47.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.673 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.673 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.854 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [220ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@451900251(state=PENDING, result=null, callback=(continuation of Deferred@1131846036 after retry RPC after error@1252916002) -> (continuation of Deferred@51720626 after retry RPC after error@1652686624) -> (continuation of Deferred@357331165 after retry RPC after error@815656381) -> (continuation of Deferred@439337661 after retry RPC after error@2000311847) -> (continuation of Deferred@25565829 after retry RPC after error@1579810573) -> (continuation of Deferred@989328479 after retry RPC after error@525932104) -> (continuation of Deferred@783971604 after retry RPC after error@1073358425), errback=(continuation of Deferred@1131846036 after retry RPC after error@1252916002) -> (continuation of Deferred@51720626 after retry RPC after error@1652686624) -> (continuation of Deferred@357331165 after retry RPC after error@815656381) -> (continuation of Deferred@439337661 after retry RPC after error@2000311847) -> (continuation of Deferred@25565829 after retry RPC after error@1579810573) -> (continuation of Deferred@989328479 after retry RPC after error@525932104) -> (continuation of Deferred@783971604 after retry RPC after error@1073358425)))
03:03:47.854 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [220ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@451900251(state=PENDING, result=null, callback=(continuation of Deferred@1131846036 after retry RPC after error@1252916002) -> (continuation of Deferred@51720626 after retry RPC after error@1652686624) -> (continuation of Deferred@357331165 after retry RPC after error@815656381) -> (continuation of Deferred@439337661 after retry RPC after error@2000311847) -> (continuation of Deferred@25565829 after retry RPC after error@1579810573) -> (continuation of Deferred@989328479 after retry RPC after error@525932104) -> (continuation of Deferred@783971604 after retry RPC after error@1073358425), errback=(continuation of Deferred@1131846036 after retry RPC after error@1252916002) -> (continuation of Deferred@51720626 after retry RPC after error@1652686624) -> (continuation of Deferred@357331165 after retry RPC after error@815656381) -> (continuation of Deferred@439337661 after retry RPC after error@2000311847) -> (continuation of Deferred@25565829 after retry RPC after error@1579810573) -> (continuation of Deferred@989328479 after retry RPC after error@525932104) -> (continuation of Deferred@783971604 after retry RPC after error@1073358425)))
03:03:47.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.873 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:47.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:47.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:47.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:47.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:47.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.074 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1775199514(state=PENDING, result=null, callback=(continuation of Deferred@1258176806 after retry RPC after error@2018701355) -> (continuation of Deferred@2019635449 after retry RPC after error@1779723610) -> (continuation of Deferred@1381758543 after retry RPC after error@1038583687) -> (continuation of Deferred@1282448954 after retry RPC after error@88560723) -> (continuation of Deferred@136524602 after retry RPC after error@167331334) -> (continuation of Deferred@1102092365 after retry RPC after error@2120923620) -> (continuation of Deferred@1866592240 after retry RPC after error@751521128) -> (continuation of Deferred@1769790028 after retry RPC after error@1123216158), errback=(continuation of Deferred@1258176806 after retry RPC after error@2018701355) -> (continuation of Deferred@2019635449 after retry RPC after error@1779723610) -> (continuation of Deferred@1381758543 after retry RPC after error@1038583687) -> (continuation of Deferred@1282448954 after retry RPC after error@88560723) -> (continuation of Deferred@136524602 after retry RPC after error@167331334) -> (continuation of Deferred@1102092365 after retry RPC after error@2120923620) -> (continuation of Deferred@1866592240 after retry RPC after error@751521128) -> (continuation of Deferred@1769790028 after retry RPC after error@1123216158)))
03:03:48.074 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1775199514(state=PENDING, result=null, callback=(continuation of Deferred@1258176806 after retry RPC after error@2018701355) -> (continuation of Deferred@2019635449 after retry RPC after error@1779723610) -> (continuation of Deferred@1381758543 after retry RPC after error@1038583687) -> (continuation of Deferred@1282448954 after retry RPC after error@88560723) -> (continuation of Deferred@136524602 after retry RPC after error@167331334) -> (continuation of Deferred@1102092365 after retry RPC after error@2120923620) -> (continuation of Deferred@1866592240 after retry RPC after error@751521128) -> (continuation of Deferred@1769790028 after retry RPC after error@1123216158), errback=(continuation of Deferred@1258176806 after retry RPC after error@2018701355) -> (continuation of Deferred@2019635449 after retry RPC after error@1779723610) -> (continuation of Deferred@1381758543 after retry RPC after error@1038583687) -> (continuation of Deferred@1282448954 after retry RPC after error@88560723) -> (continuation of Deferred@136524602 after retry RPC after error@167331334) -> (continuation of Deferred@1102092365 after retry RPC after error@2120923620) -> (continuation of Deferred@1866592240 after retry RPC after error@751521128) -> (continuation of Deferred@1769790028 after retry RPC after error@1123216158)))
03:03:48.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.351 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@915893646(state=PENDING, result=null, callback=(continuation of Deferred@337569266 after retry RPC after error@1505458540) -> (continuation of Deferred@1228392451 after retry RPC after error@1145120694) -> (continuation of Deferred@663976889 after retry RPC after error@627119678) -> (continuation of Deferred@1390519617 after retry RPC after error@525354044) -> (continuation of Deferred@597863132 after retry RPC after error@249534576) -> (continuation of Deferred@1363930951 after retry RPC after error@1277632758) -> (continuation of Deferred@2061862884 after retry RPC after error@143367376), errback=(continuation of Deferred@337569266 after retry RPC after error@1505458540) -> (continuation of Deferred@1228392451 after retry RPC after error@1145120694) -> (continuation of Deferred@663976889 after retry RPC after error@627119678) -> (continuation of Deferred@1390519617 after retry RPC after error@525354044) -> (continuation of Deferred@597863132 after retry RPC after error@249534576) -> (continuation of Deferred@1363930951 after retry RPC after error@1277632758) -> (continuation of Deferred@2061862884 after retry RPC after error@143367376)))
03:03:48.352 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:48.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.513 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1985450128(state=PENDING, result=null, callback=(continuation of Deferred@1089586470 after retry RPC after error@137537203) -> (continuation of Deferred@29388911 after retry RPC after error@1264165847) -> (continuation of Deferred@97490799 after retry RPC after error@416661083) -> (continuation of Deferred@437577254 after retry RPC after error@475380136) -> (continuation of Deferred@1708663202 after retry RPC after error@1355447332) -> (continuation of Deferred@1834128845 after retry RPC after error@1551628598), errback=(continuation of Deferred@1089586470 after retry RPC after error@137537203) -> (continuation of Deferred@29388911 after retry RPC after error@1264165847) -> (continuation of Deferred@97490799 after retry RPC after error@416661083) -> (continuation of Deferred@437577254 after retry RPC after error@475380136) -> (continuation of Deferred@1708663202 after retry RPC after error@1355447332) -> (continuation of Deferred@1834128845 after retry RPC after error@1551628598)))
03:03:48.514 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1985450128(state=PENDING, result=null, callback=(continuation of Deferred@1089586470 after retry RPC after error@137537203) -> (continuation of Deferred@29388911 after retry RPC after error@1264165847) -> (continuation of Deferred@97490799 after retry RPC after error@416661083) -> (continuation of Deferred@437577254 after retry RPC after error@475380136) -> (continuation of Deferred@1708663202 after retry RPC after error@1355447332) -> (continuation of Deferred@1834128845 after retry RPC after error@1551628598), errback=(continuation of Deferred@1089586470 after retry RPC after error@137537203) -> (continuation of Deferred@29388911 after retry RPC after error@1264165847) -> (continuation of Deferred@97490799 after retry RPC after error@416661083) -> (continuation of Deferred@437577254 after retry RPC after error@475380136) -> (continuation of Deferred@1708663202 after retry RPC after error@1355447332) -> (continuation of Deferred@1834128845 after retry RPC after error@1551628598)))
03:03:48.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.553 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.754 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1147781777(state=PENDING, result=null, callback=(continuation of Deferred@1077687519 after retry RPC after error@1953257003) -> (continuation of Deferred@1725793544 after retry RPC after error@1823129096) -> (continuation of Deferred@1301407118 after retry RPC after error@1532636953) -> (continuation of Deferred@1361380543 after retry RPC after error@1749461613) -> (continuation of Deferred@716803966 after retry RPC after error@546394071) -> (continuation of Deferred@1508423558 after retry RPC after error@77622036) -> (continuation of Deferred@1652872833 after retry RPC after error@1220662001) -> (continuation of Deferred@1418266856 after retry RPC after error@1400254886), errback=(continuation of Deferred@1077687519 after retry RPC after error@1953257003) -> (continuation of Deferred@1725793544 after retry RPC after error@1823129096) -> (continuation of Deferred@1301407118 after retry RPC after error@1532636953) -> (continuation of Deferred@1361380543 after retry RPC after error@1749461613) -> (continuation of Deferred@716803966 after retry RPC after error@546394071) -> (continuation of Deferred@1508423558 after retry RPC after error@77622036) -> (continuation of Deferred@1652872833 after retry RPC after error@1220662001) -> (continuation of Deferred@1418266856 after retry RPC after error@1400254886)))
03:03:48.754 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1147781777(state=PENDING, result=null, callback=(continuation of Deferred@1077687519 after retry RPC after error@1953257003) -> (continuation of Deferred@1725793544 after retry RPC after error@1823129096) -> (continuation of Deferred@1301407118 after retry RPC after error@1532636953) -> (continuation of Deferred@1361380543 after retry RPC after error@1749461613) -> (continuation of Deferred@716803966 after retry RPC after error@546394071) -> (continuation of Deferred@1508423558 after retry RPC after error@77622036) -> (continuation of Deferred@1652872833 after retry RPC after error@1220662001) -> (continuation of Deferred@1418266856 after retry RPC after error@1400254886), errback=(continuation of Deferred@1077687519 after retry RPC after error@1953257003) -> (continuation of Deferred@1725793544 after retry RPC after error@1823129096) -> (continuation of Deferred@1301407118 after retry RPC after error@1532636953) -> (continuation of Deferred@1361380543 after retry RPC after error@1749461613) -> (continuation of Deferred@716803966 after retry RPC after error@546394071) -> (continuation of Deferred@1508423558 after retry RPC after error@77622036) -> (continuation of Deferred@1652872833 after retry RPC after error@1220662001) -> (continuation of Deferred@1418266856 after retry RPC after error@1400254886)))
03:03:48.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.873 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.874 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.874 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.874 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.874 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.874 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.874 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.875 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:48.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:48.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:48.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:48.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:48.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:48.942967 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:48.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:48.945469 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:48.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:48.947003 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:49.031 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1148070051(state=PENDING, result=null, callback=(continuation of Deferred@1786715087 after retry RPC after error@2020886388) -> (continuation of Deferred@520272595 after retry RPC after error@714908320) -> (continuation of Deferred@942634335 after retry RPC after error@1648020169) -> (continuation of Deferred@1759780991 after retry RPC after error@656545377) -> (continuation of Deferred@1684277847 after retry RPC after error@1346168976) -> (continuation of Deferred@420666672 after retry RPC after error@775176557) -> (continuation of Deferred@1783021646 after retry RPC after error@1569316350), errback=(continuation of Deferred@1786715087 after retry RPC after error@2020886388) -> (continuation of Deferred@520272595 after retry RPC after error@714908320) -> (continuation of Deferred@942634335 after retry RPC after error@1648020169) -> (continuation of Deferred@1759780991 after retry RPC after error@656545377) -> (continuation of Deferred@1684277847 after retry RPC after error@1346168976) -> (continuation of Deferred@420666672 after retry RPC after error@775176557) -> (continuation of Deferred@1783021646 after retry RPC after error@1569316350)))
03:03:49.032 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:49.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.193 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@669009885(state=PENDING, result=null, callback=(continuation of Deferred@701470588 after retry RPC after error@767998340) -> (continuation of Deferred@1944384498 after retry RPC after error@1453296892) -> (continuation of Deferred@62821668 after retry RPC after error@464262734) -> (continuation of Deferred@1333992874 after retry RPC after error@1699844298) -> (continuation of Deferred@305213169 after retry RPC after error@1775518879) -> (continuation of Deferred@1047358018 after retry RPC after error@1473473157), errback=(continuation of Deferred@701470588 after retry RPC after error@767998340) -> (continuation of Deferred@1944384498 after retry RPC after error@1453296892) -> (continuation of Deferred@62821668 after retry RPC after error@464262734) -> (continuation of Deferred@1333992874 after retry RPC after error@1699844298) -> (continuation of Deferred@305213169 after retry RPC after error@1775518879) -> (continuation of Deferred@1047358018 after retry RPC after error@1473473157)))
03:03:49.194 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@669009885(state=PENDING, result=null, callback=(continuation of Deferred@701470588 after retry RPC after error@767998340) -> (continuation of Deferred@1944384498 after retry RPC after error@1453296892) -> (continuation of Deferred@62821668 after retry RPC after error@464262734) -> (continuation of Deferred@1333992874 after retry RPC after error@1699844298) -> (continuation of Deferred@305213169 after retry RPC after error@1775518879) -> (continuation of Deferred@1047358018 after retry RPC after error@1473473157), errback=(continuation of Deferred@701470588 after retry RPC after error@767998340) -> (continuation of Deferred@1944384498 after retry RPC after error@1453296892) -> (continuation of Deferred@62821668 after retry RPC after error@464262734) -> (continuation of Deferred@1333992874 after retry RPC after error@1699844298) -> (continuation of Deferred@305213169 after retry RPC after error@1775518879) -> (continuation of Deferred@1047358018 after retry RPC after error@1473473157)))
03:03:49.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.414 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1904692975(state=PENDING, result=null, callback=(continuation of Deferred@954641331 after retry RPC after error@668812180) -> (continuation of Deferred@1788154146 after retry RPC after error@556714665) -> (continuation of Deferred@748181004 after retry RPC after error@1628725736) -> (continuation of Deferred@592077033 after retry RPC after error@1961266603) -> (continuation of Deferred@446733581 after retry RPC after error@1492755600) -> (continuation of Deferred@675596878 after retry RPC after error@1754235192) -> (continuation of Deferred@1993941457 after retry RPC after error@1670497982), errback=(continuation of Deferred@954641331 after retry RPC after error@668812180) -> (continuation of Deferred@1788154146 after retry RPC after error@556714665) -> (continuation of Deferred@748181004 after retry RPC after error@1628725736) -> (continuation of Deferred@592077033 after retry RPC after error@1961266603) -> (continuation of Deferred@446733581 after retry RPC after error@1492755600) -> (continuation of Deferred@675596878 after retry RPC after error@1754235192) -> (continuation of Deferred@1993941457 after retry RPC after error@1670497982)))
03:03:49.414 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1904692975(state=PENDING, result=null, callback=(continuation of Deferred@954641331 after retry RPC after error@668812180) -> (continuation of Deferred@1788154146 after retry RPC after error@556714665) -> (continuation of Deferred@748181004 after retry RPC after error@1628725736) -> (continuation of Deferred@592077033 after retry RPC after error@1961266603) -> (continuation of Deferred@446733581 after retry RPC after error@1492755600) -> (continuation of Deferred@675596878 after retry RPC after error@1754235192) -> (continuation of Deferred@1993941457 after retry RPC after error@1670497982), errback=(continuation of Deferred@954641331 after retry RPC after error@668812180) -> (continuation of Deferred@1788154146 after retry RPC after error@556714665) -> (continuation of Deferred@748181004 after retry RPC after error@1628725736) -> (continuation of Deferred@592077033 after retry RPC after error@1961266603) -> (continuation of Deferred@446733581 after retry RPC after error@1492755600) -> (continuation of Deferred@675596878 after retry RPC after error@1754235192) -> (continuation of Deferred@1993941457 after retry RPC after error@1670497982)))
03:03:49.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.614 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1503230027(state=PENDING, result=null, callback=(continuation of Deferred@773930484 after retry RPC after error@1419062448) -> (continuation of Deferred@838022497 after retry RPC after error@599410740) -> (continuation of Deferred@80944938 after retry RPC after error@2044613548) -> (continuation of Deferred@1361983947 after retry RPC after error@539242546) -> (continuation of Deferred@1007833427 after retry RPC after error@937043534) -> (continuation of Deferred@396013855 after retry RPC after error@373208702) -> (continuation of Deferred@940107043 after retry RPC after error@1585859642), errback=(continuation of Deferred@773930484 after retry RPC after error@1419062448) -> (continuation of Deferred@838022497 after retry RPC after error@599410740) -> (continuation of Deferred@80944938 after retry RPC after error@2044613548) -> (continuation of Deferred@1361983947 after retry RPC after error@539242546) -> (continuation of Deferred@1007833427 after retry RPC after error@937043534) -> (continuation of Deferred@396013855 after retry RPC after error@373208702) -> (continuation of Deferred@940107043 after retry RPC after error@1585859642)))
03:03:49.614 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1503230027(state=PENDING, result=null, callback=(continuation of Deferred@773930484 after retry RPC after error@1419062448) -> (continuation of Deferred@838022497 after retry RPC after error@599410740) -> (continuation of Deferred@80944938 after retry RPC after error@2044613548) -> (continuation of Deferred@1361983947 after retry RPC after error@539242546) -> (continuation of Deferred@1007833427 after retry RPC after error@937043534) -> (continuation of Deferred@396013855 after retry RPC after error@373208702) -> (continuation of Deferred@940107043 after retry RPC after error@1585859642), errback=(continuation of Deferred@773930484 after retry RPC after error@1419062448) -> (continuation of Deferred@838022497 after retry RPC after error@599410740) -> (continuation of Deferred@80944938 after retry RPC after error@2044613548) -> (continuation of Deferred@1361983947 after retry RPC after error@539242546) -> (continuation of Deferred@1007833427 after retry RPC after error@937043534) -> (continuation of Deferred@396013855 after retry RPC after error@373208702) -> (continuation of Deferred@940107043 after retry RPC after error@1585859642)))
03:03:49.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1614324963(state=PENDING, result=null, callback=(continuation of Deferred@470918210 after retry RPC after error@1096320165) -> (continuation of Deferred@1357992100 after retry RPC after error@1967909349) -> (continuation of Deferred@1684627642 after retry RPC after error@1628552510) -> (continuation of Deferred@994031048 after retry RPC after error@286357387) -> (continuation of Deferred@1045604771 after retry RPC after error@1380326170) -> (continuation of Deferred@336684399 after retry RPC after error@237430089), errback=(continuation of Deferred@470918210 after retry RPC after error@1096320165) -> (continuation of Deferred@1357992100 after retry RPC after error@1967909349) -> (continuation of Deferred@1684627642 after retry RPC after error@1628552510) -> (continuation of Deferred@994031048 after retry RPC after error@286357387) -> (continuation of Deferred@1045604771 after retry RPC after error@1380326170) -> (continuation of Deferred@336684399 after retry RPC after error@237430089)))
03:03:49.794 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1614324963(state=PENDING, result=null, callback=(continuation of Deferred@470918210 after retry RPC after error@1096320165) -> (continuation of Deferred@1357992100 after retry RPC after error@1967909349) -> (continuation of Deferred@1684627642 after retry RPC after error@1628552510) -> (continuation of Deferred@994031048 after retry RPC after error@286357387) -> (continuation of Deferred@1045604771 after retry RPC after error@1380326170) -> (continuation of Deferred@336684399 after retry RPC after error@237430089), errback=(continuation of Deferred@470918210 after retry RPC after error@1096320165) -> (continuation of Deferred@1357992100 after retry RPC after error@1967909349) -> (continuation of Deferred@1684627642 after retry RPC after error@1628552510) -> (continuation of Deferred@994031048 after retry RPC after error@286357387) -> (continuation of Deferred@1045604771 after retry RPC after error@1380326170) -> (continuation of Deferred@336684399 after retry RPC after error@237430089)))
03:03:49.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.876 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:49.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:49.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:49.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:49.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:49.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.072 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [65ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1538551500(state=PENDING, result=null, callback=(continuation of Deferred@670174740 after retry RPC after error@663038657) -> (continuation of Deferred@1392865142 after retry RPC after error@725592532) -> (continuation of Deferred@470454281 after retry RPC after error@316105323) -> (continuation of Deferred@1792131480 after retry RPC after error@1022800927) -> (continuation of Deferred@1949622608 after retry RPC after error@1566118279) -> (continuation of Deferred@962647733 after retry RPC after error@817875914) -> (continuation of Deferred@1666351852 after retry RPC after error@1989308968), errback=(continuation of Deferred@670174740 after retry RPC after error@663038657) -> (continuation of Deferred@1392865142 after retry RPC after error@725592532) -> (continuation of Deferred@470454281 after retry RPC after error@316105323) -> (continuation of Deferred@1792131480 after retry RPC after error@1022800927) -> (continuation of Deferred@1949622608 after retry RPC after error@1566118279) -> (continuation of Deferred@962647733 after retry RPC after error@817875914) -> (continuation of Deferred@1666351852 after retry RPC after error@1989308968)))
03:03:50.072 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:50.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.254 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1117892086(state=PENDING, result=null, callback=(continuation of Deferred@992301143 after retry RPC after error@713894916) -> (continuation of Deferred@1054166555 after retry RPC after error@1901369898) -> (continuation of Deferred@1617091519 after retry RPC after error@592193211) -> (continuation of Deferred@930943327 after retry RPC after error@1647134800) -> (continuation of Deferred@342153353 after retry RPC after error@1449086272) -> (continuation of Deferred@167201085 after retry RPC after error@26279835), errback=(continuation of Deferred@992301143 after retry RPC after error@713894916) -> (continuation of Deferred@1054166555 after retry RPC after error@1901369898) -> (continuation of Deferred@1617091519 after retry RPC after error@592193211) -> (continuation of Deferred@930943327 after retry RPC after error@1647134800) -> (continuation of Deferred@342153353 after retry RPC after error@1449086272) -> (continuation of Deferred@167201085 after retry RPC after error@26279835)))
03:03:50.254 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1117892086(state=PENDING, result=null, callback=(continuation of Deferred@992301143 after retry RPC after error@713894916) -> (continuation of Deferred@1054166555 after retry RPC after error@1901369898) -> (continuation of Deferred@1617091519 after retry RPC after error@592193211) -> (continuation of Deferred@930943327 after retry RPC after error@1647134800) -> (continuation of Deferred@342153353 after retry RPC after error@1449086272) -> (continuation of Deferred@167201085 after retry RPC after error@26279835), errback=(continuation of Deferred@992301143 after retry RPC after error@713894916) -> (continuation of Deferred@1054166555 after retry RPC after error@1901369898) -> (continuation of Deferred@1617091519 after retry RPC after error@592193211) -> (continuation of Deferred@930943327 after retry RPC after error@1647134800) -> (continuation of Deferred@342153353 after retry RPC after error@1449086272) -> (continuation of Deferred@167201085 after retry RPC after error@26279835)))
03:03:50.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.531 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1454678136(state=PENDING, result=null, callback=(continuation of Deferred@108221092 after retry RPC after error@1249431619) -> (continuation of Deferred@1112079955 after retry RPC after error@831271170) -> (continuation of Deferred@771142629 after retry RPC after error@553682128) -> (continuation of Deferred@608132293 after retry RPC after error@1961586017) -> (continuation of Deferred@1813680514 after retry RPC after error@1226419925) -> (continuation of Deferred@396237383 after retry RPC after error@942796208) -> (continuation of Deferred@10481077 after retry RPC after error@83137653), errback=(continuation of Deferred@108221092 after retry RPC after error@1249431619) -> (continuation of Deferred@1112079955 after retry RPC after error@831271170) -> (continuation of Deferred@771142629 after retry RPC after error@553682128) -> (continuation of Deferred@608132293 after retry RPC after error@1961586017) -> (continuation of Deferred@1813680514 after retry RPC after error@1226419925) -> (continuation of Deferred@396237383 after retry RPC after error@942796208) -> (continuation of Deferred@10481077 after retry RPC after error@83137653)))
03:03:50.532 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:50.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.714 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@590693137(state=PENDING, result=null, callback=(continuation of Deferred@1962328934 after retry RPC after error@1481258500) -> (continuation of Deferred@1752031190 after retry RPC after error@1632183297) -> (continuation of Deferred@593897149 after retry RPC after error@714460867) -> (continuation of Deferred@746600232 after retry RPC after error@71395221) -> (continuation of Deferred@369162862 after retry RPC after error@308792763) -> (continuation of Deferred@1072927144 after retry RPC after error@813769740), errback=(continuation of Deferred@1962328934 after retry RPC after error@1481258500) -> (continuation of Deferred@1752031190 after retry RPC after error@1632183297) -> (continuation of Deferred@593897149 after retry RPC after error@714460867) -> (continuation of Deferred@746600232 after retry RPC after error@71395221) -> (continuation of Deferred@369162862 after retry RPC after error@308792763) -> (continuation of Deferred@1072927144 after retry RPC after error@813769740)))
03:03:50.715 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@590693137(state=PENDING, result=null, callback=(continuation of Deferred@1962328934 after retry RPC after error@1481258500) -> (continuation of Deferred@1752031190 after retry RPC after error@1632183297) -> (continuation of Deferred@593897149 after retry RPC after error@714460867) -> (continuation of Deferred@746600232 after retry RPC after error@71395221) -> (continuation of Deferred@369162862 after retry RPC after error@308792763) -> (continuation of Deferred@1072927144 after retry RPC after error@813769740), errback=(continuation of Deferred@1962328934 after retry RPC after error@1481258500) -> (continuation of Deferred@1752031190 after retry RPC after error@1632183297) -> (continuation of Deferred@593897149 after retry RPC after error@714460867) -> (continuation of Deferred@746600232 after retry RPC after error@71395221) -> (continuation of Deferred@369162862 after retry RPC after error@308792763) -> (continuation of Deferred@1072927144 after retry RPC after error@813769740)))
03:03:50.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.914 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1744493175(state=PENDING, result=null, callback=(continuation of Deferred@1162211468 after retry RPC after error@567693550) -> (continuation of Deferred@1282599783 after retry RPC after error@2104142332) -> (continuation of Deferred@398587725 after retry RPC after error@75727995) -> (continuation of Deferred@1601273439 after retry RPC after error@2004617476) -> (continuation of Deferred@1864364857 after retry RPC after error@2007438077) -> (continuation of Deferred@594545104 after retry RPC after error@877854867) -> (continuation of Deferred@616934171 after retry RPC after error@1798993284), errback=(continuation of Deferred@1162211468 after retry RPC after error@567693550) -> (continuation of Deferred@1282599783 after retry RPC after error@2104142332) -> (continuation of Deferred@398587725 after retry RPC after error@75727995) -> (continuation of Deferred@1601273439 after retry RPC after error@2004617476) -> (continuation of Deferred@1864364857 after retry RPC after error@2007438077) -> (continuation of Deferred@594545104 after retry RPC after error@877854867) -> (continuation of Deferred@616934171 after retry RPC after error@1798993284)))
03:03:50.914 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1744493175(state=PENDING, result=null, callback=(continuation of Deferred@1162211468 after retry RPC after error@567693550) -> (continuation of Deferred@1282599783 after retry RPC after error@2104142332) -> (continuation of Deferred@398587725 after retry RPC after error@75727995) -> (continuation of Deferred@1601273439 after retry RPC after error@2004617476) -> (continuation of Deferred@1864364857 after retry RPC after error@2007438077) -> (continuation of Deferred@594545104 after retry RPC after error@877854867) -> (continuation of Deferred@616934171 after retry RPC after error@1798993284), errback=(continuation of Deferred@1162211468 after retry RPC after error@567693550) -> (continuation of Deferred@1282599783 after retry RPC after error@2104142332) -> (continuation of Deferred@398587725 after retry RPC after error@75727995) -> (continuation of Deferred@1601273439 after retry RPC after error@2004617476) -> (continuation of Deferred@1864364857 after retry RPC after error@2007438077) -> (continuation of Deferred@594545104 after retry RPC after error@877854867) -> (continuation of Deferred@616934171 after retry RPC after error@1798993284)))
03:03:50.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:50.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:50.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:50.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:50.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:50.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:51.099408 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:03:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:51.099680 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:51.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.191 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1308596582(state=PENDING, result=null, callback=(continuation of Deferred@1981038372 after retry RPC after error@402901470) -> (continuation of Deferred@432022897 after retry RPC after error@675658630) -> (continuation of Deferred@2147441845 after retry RPC after error@2041853869) -> (continuation of Deferred@1632071239 after retry RPC after error@1683710050) -> (continuation of Deferred@1631879848 after retry RPC after error@957732978) -> (continuation of Deferred@1677376577 after retry RPC after error@1545439276) -> (continuation of Deferred@420024517 after retry RPC after error@239635934) -> (continuation of Deferred@946221282 after retry RPC after error@143299370), errback=(continuation of Deferred@1981038372 after retry RPC after error@402901470) -> (continuation of Deferred@432022897 after retry RPC after error@675658630) -> (continuation of Deferred@2147441845 after retry RPC after error@2041853869) -> (continuation of Deferred@1632071239 after retry RPC after error@1683710050) -> (continuation of Deferred@1631879848 after retry RPC after error@957732978) -> (continuation of Deferred@1677376577 after retry RPC after error@1545439276) -> (continuation of Deferred@420024517 after retry RPC after error@239635934) -> (continuation of Deferred@946221282 after retry RPC after error@143299370)))
03:03:51.191 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:51.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.375 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@326573776(state=PENDING, result=null, callback=(continuation of Deferred@1905464371 after retry RPC after error@1706282561) -> (continuation of Deferred@1953334137 after retry RPC after error@1606914746) -> (continuation of Deferred@907814909 after retry RPC after error@1294350009) -> (continuation of Deferred@1430782511 after retry RPC after error@913905790) -> (continuation of Deferred@1628304003 after retry RPC after error@501388261) -> (continuation of Deferred@145666828 after retry RPC after error@1995104399) -> (continuation of Deferred@2021494075 after retry RPC after error@1896436842) -> (continuation of Deferred@205696916 after retry RPC after error@755877899), errback=(continuation of Deferred@1905464371 after retry RPC after error@1706282561) -> (continuation of Deferred@1953334137 after retry RPC after error@1606914746) -> (continuation of Deferred@907814909 after retry RPC after error@1294350009) -> (continuation of Deferred@1430782511 after retry RPC after error@913905790) -> (continuation of Deferred@1628304003 after retry RPC after error@501388261) -> (continuation of Deferred@145666828 after retry RPC after error@1995104399) -> (continuation of Deferred@2021494075 after retry RPC after error@1896436842) -> (continuation of Deferred@205696916 after retry RPC after error@755877899)))
03:03:51.375 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@326573776(state=PENDING, result=null, callback=(continuation of Deferred@1905464371 after retry RPC after error@1706282561) -> (continuation of Deferred@1953334137 after retry RPC after error@1606914746) -> (continuation of Deferred@907814909 after retry RPC after error@1294350009) -> (continuation of Deferred@1430782511 after retry RPC after error@913905790) -> (continuation of Deferred@1628304003 after retry RPC after error@501388261) -> (continuation of Deferred@145666828 after retry RPC after error@1995104399) -> (continuation of Deferred@2021494075 after retry RPC after error@1896436842) -> (continuation of Deferred@205696916 after retry RPC after error@755877899), errback=(continuation of Deferred@1905464371 after retry RPC after error@1706282561) -> (continuation of Deferred@1953334137 after retry RPC after error@1606914746) -> (continuation of Deferred@907814909 after retry RPC after error@1294350009) -> (continuation of Deferred@1430782511 after retry RPC after error@913905790) -> (continuation of Deferred@1628304003 after retry RPC after error@501388261) -> (continuation of Deferred@145666828 after retry RPC after error@1995104399) -> (continuation of Deferred@2021494075 after retry RPC after error@1896436842) -> (continuation of Deferred@205696916 after retry RPC after error@755877899)))
03:03:51.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.554 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2014593445(state=PENDING, result=null, callback=(continuation of Deferred@613546524 after retry RPC after error@1316381601) -> (continuation of Deferred@167827092 after retry RPC after error@1346615064) -> (continuation of Deferred@2066347542 after retry RPC after error@104851897) -> (continuation of Deferred@168246172 after retry RPC after error@1194440371) -> (continuation of Deferred@1931243781 after retry RPC after error@169291006) -> (continuation of Deferred@973024763 after retry RPC after error@1932623961), errback=(continuation of Deferred@613546524 after retry RPC after error@1316381601) -> (continuation of Deferred@167827092 after retry RPC after error@1346615064) -> (continuation of Deferred@2066347542 after retry RPC after error@104851897) -> (continuation of Deferred@168246172 after retry RPC after error@1194440371) -> (continuation of Deferred@1931243781 after retry RPC after error@169291006) -> (continuation of Deferred@973024763 after retry RPC after error@1932623961)))
03:03:51.555 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2014593445(state=PENDING, result=null, callback=(continuation of Deferred@613546524 after retry RPC after error@1316381601) -> (continuation of Deferred@167827092 after retry RPC after error@1346615064) -> (continuation of Deferred@2066347542 after retry RPC after error@104851897) -> (continuation of Deferred@168246172 after retry RPC after error@1194440371) -> (continuation of Deferred@1931243781 after retry RPC after error@169291006) -> (continuation of Deferred@973024763 after retry RPC after error@1932623961), errback=(continuation of Deferred@613546524 after retry RPC after error@1316381601) -> (continuation of Deferred@167827092 after retry RPC after error@1346615064) -> (continuation of Deferred@2066347542 after retry RPC after error@104851897) -> (continuation of Deferred@168246172 after retry RPC after error@1194440371) -> (continuation of Deferred@1931243781 after retry RPC after error@169291006) -> (continuation of Deferred@973024763 after retry RPC after error@1932623961)))
03:03:51.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.814 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1195960037(state=PENDING, result=null, callback=(continuation of Deferred@12274951 after retry RPC after error@1772288532) -> (continuation of Deferred@1009600333 after retry RPC after error@1020457443) -> (continuation of Deferred@1717256203 after retry RPC after error@416773262) -> (continuation of Deferred@1318473718 after retry RPC after error@1709240878) -> (continuation of Deferred@1502068097 after retry RPC after error@2052571451) -> (continuation of Deferred@105039992 after retry RPC after error@1683045148) -> (continuation of Deferred@6220194 after retry RPC after error@1145351819) -> (continuation of Deferred@1414901619 after retry RPC after error@959575870), errback=(continuation of Deferred@12274951 after retry RPC after error@1772288532) -> (continuation of Deferred@1009600333 after retry RPC after error@1020457443) -> (continuation of Deferred@1717256203 after retry RPC after error@416773262) -> (continuation of Deferred@1318473718 after retry RPC after error@1709240878) -> (continuation of Deferred@1502068097 after retry RPC after error@2052571451) -> (continuation of Deferred@105039992 after retry RPC after error@1683045148) -> (continuation of Deferred@6220194 after retry RPC after error@1145351819) -> (continuation of Deferred@1414901619 after retry RPC after error@959575870)))
03:03:51.814 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1195960037(state=PENDING, result=null, callback=(continuation of Deferred@12274951 after retry RPC after error@1772288532) -> (continuation of Deferred@1009600333 after retry RPC after error@1020457443) -> (continuation of Deferred@1717256203 after retry RPC after error@416773262) -> (continuation of Deferred@1318473718 after retry RPC after error@1709240878) -> (continuation of Deferred@1502068097 after retry RPC after error@2052571451) -> (continuation of Deferred@105039992 after retry RPC after error@1683045148) -> (continuation of Deferred@6220194 after retry RPC after error@1145351819) -> (continuation of Deferred@1414901619 after retry RPC after error@959575870), errback=(continuation of Deferred@12274951 after retry RPC after error@1772288532) -> (continuation of Deferred@1009600333 after retry RPC after error@1020457443) -> (continuation of Deferred@1717256203 after retry RPC after error@416773262) -> (continuation of Deferred@1318473718 after retry RPC after error@1709240878) -> (continuation of Deferred@1502068097 after retry RPC after error@2052571451) -> (continuation of Deferred@105039992 after retry RPC after error@1683045148) -> (continuation of Deferred@6220194 after retry RPC after error@1145351819) -> (continuation of Deferred@1414901619 after retry RPC after error@959575870)))
03:03:51.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:51.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:51.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:51.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:51.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:51.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.074 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@580715505(state=PENDING, result=null, callback=(continuation of Deferred@1354242078 after retry RPC after error@1820922481) -> (continuation of Deferred@402862013 after retry RPC after error@1380660679) -> (continuation of Deferred@1109634936 after retry RPC after error@1721575383) -> (continuation of Deferred@1734509182 after retry RPC after error@1821140568) -> (continuation of Deferred@864771294 after retry RPC after error@1009215207) -> (continuation of Deferred@1327850095 after retry RPC after error@1083470709) -> (continuation of Deferred@697350677 after retry RPC after error@780661150), errback=(continuation of Deferred@1354242078 after retry RPC after error@1820922481) -> (continuation of Deferred@402862013 after retry RPC after error@1380660679) -> (continuation of Deferred@1109634936 after retry RPC after error@1721575383) -> (continuation of Deferred@1734509182 after retry RPC after error@1821140568) -> (continuation of Deferred@864771294 after retry RPC after error@1009215207) -> (continuation of Deferred@1327850095 after retry RPC after error@1083470709) -> (continuation of Deferred@697350677 after retry RPC after error@780661150)))
03:03:52.074 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@580715505(state=PENDING, result=null, callback=(continuation of Deferred@1354242078 after retry RPC after error@1820922481) -> (continuation of Deferred@402862013 after retry RPC after error@1380660679) -> (continuation of Deferred@1109634936 after retry RPC after error@1721575383) -> (continuation of Deferred@1734509182 after retry RPC after error@1821140568) -> (continuation of Deferred@864771294 after retry RPC after error@1009215207) -> (continuation of Deferred@1327850095 after retry RPC after error@1083470709) -> (continuation of Deferred@697350677 after retry RPC after error@780661150), errback=(continuation of Deferred@1354242078 after retry RPC after error@1820922481) -> (continuation of Deferred@402862013 after retry RPC after error@1380660679) -> (continuation of Deferred@1109634936 after retry RPC after error@1721575383) -> (continuation of Deferred@1734509182 after retry RPC after error@1821140568) -> (continuation of Deferred@864771294 after retry RPC after error@1009215207) -> (continuation of Deferred@1327850095 after retry RPC after error@1083470709) -> (continuation of Deferred@697350677 after retry RPC after error@780661150)))
03:03:52.075 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.253 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.253 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.314 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1006253052(state=PENDING, result=null, callback=(continuation of Deferred@1743832277 after retry RPC after error@727774356) -> (continuation of Deferred@985141530 after retry RPC after error@1354215522) -> (continuation of Deferred@822087885 after retry RPC after error@493393453) -> (continuation of Deferred@1730307163 after retry RPC after error@175897725) -> (continuation of Deferred@994837383 after retry RPC after error@1657549629) -> (continuation of Deferred@381195139 after retry RPC after error@1140133181) -> (continuation of Deferred@410939135 after retry RPC after error@464057110), errback=(continuation of Deferred@1743832277 after retry RPC after error@727774356) -> (continuation of Deferred@985141530 after retry RPC after error@1354215522) -> (continuation of Deferred@822087885 after retry RPC after error@493393453) -> (continuation of Deferred@1730307163 after retry RPC after error@175897725) -> (continuation of Deferred@994837383 after retry RPC after error@1657549629) -> (continuation of Deferred@381195139 after retry RPC after error@1140133181) -> (continuation of Deferred@410939135 after retry RPC after error@464057110)))
03:03:52.315 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1006253052(state=PENDING, result=null, callback=(continuation of Deferred@1743832277 after retry RPC after error@727774356) -> (continuation of Deferred@985141530 after retry RPC after error@1354215522) -> (continuation of Deferred@822087885 after retry RPC after error@493393453) -> (continuation of Deferred@1730307163 after retry RPC after error@175897725) -> (continuation of Deferred@994837383 after retry RPC after error@1657549629) -> (continuation of Deferred@381195139 after retry RPC after error@1140133181) -> (continuation of Deferred@410939135 after retry RPC after error@464057110), errback=(continuation of Deferred@1743832277 after retry RPC after error@727774356) -> (continuation of Deferred@985141530 after retry RPC after error@1354215522) -> (continuation of Deferred@822087885 after retry RPC after error@493393453) -> (continuation of Deferred@1730307163 after retry RPC after error@175897725) -> (continuation of Deferred@994837383 after retry RPC after error@1657549629) -> (continuation of Deferred@381195139 after retry RPC after error@1140133181) -> (continuation of Deferred@410939135 after retry RPC after error@464057110)))
03:03:52.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.352 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:52.397142 18477 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:03:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:52.397365 18477 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:03:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:52.398065 18477 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:03:52.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:52.399060 18217 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:03:52.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:52.400368 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:03:52.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:52.400588 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:03:52.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:52.400738 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:03:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:52.400962 18477 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:03:52.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.554 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@239688995(state=PENDING, result=null, callback=(continuation of Deferred@1942231053 after retry RPC after error@1519771519) -> (continuation of Deferred@1363983563 after retry RPC after error@363348868) -> (continuation of Deferred@1191126348 after retry RPC after error@845509835) -> (continuation of Deferred@108739752 after retry RPC after error@1360591744) -> (continuation of Deferred@1861378474 after retry RPC after error@1999143827) -> (continuation of Deferred@686387106 after retry RPC after error@1052464580) -> (continuation of Deferred@1136233608 after retry RPC after error@1891543927), errback=(continuation of Deferred@1942231053 after retry RPC after error@1519771519) -> (continuation of Deferred@1363983563 after retry RPC after error@363348868) -> (continuation of Deferred@1191126348 after retry RPC after error@845509835) -> (continuation of Deferred@108739752 after retry RPC after error@1360591744) -> (continuation of Deferred@1861378474 after retry RPC after error@1999143827) -> (continuation of Deferred@686387106 after retry RPC after error@1052464580) -> (continuation of Deferred@1136233608 after retry RPC after error@1891543927)))
03:03:52.554 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [140ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@239688995(state=PENDING, result=null, callback=(continuation of Deferred@1942231053 after retry RPC after error@1519771519) -> (continuation of Deferred@1363983563 after retry RPC after error@363348868) -> (continuation of Deferred@1191126348 after retry RPC after error@845509835) -> (continuation of Deferred@108739752 after retry RPC after error@1360591744) -> (continuation of Deferred@1861378474 after retry RPC after error@1999143827) -> (continuation of Deferred@686387106 after retry RPC after error@1052464580) -> (continuation of Deferred@1136233608 after retry RPC after error@1891543927), errback=(continuation of Deferred@1942231053 after retry RPC after error@1519771519) -> (continuation of Deferred@1363983563 after retry RPC after error@363348868) -> (continuation of Deferred@1191126348 after retry RPC after error@845509835) -> (continuation of Deferred@108739752 after retry RPC after error@1360591744) -> (continuation of Deferred@1861378474 after retry RPC after error@1999143827) -> (continuation of Deferred@686387106 after retry RPC after error@1052464580) -> (continuation of Deferred@1136233608 after retry RPC after error@1891543927)))
03:03:52.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.774 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1155212814(state=PENDING, result=null, callback=(continuation of Deferred@937601918 after retry RPC after error@900958221) -> (continuation of Deferred@199420220 after retry RPC after error@370431256) -> (continuation of Deferred@881833471 after retry RPC after error@492150915) -> (continuation of Deferred@31588636 after retry RPC after error@904538285) -> (continuation of Deferred@2088981410 after retry RPC after error@1281533193) -> (continuation of Deferred@1570350968 after retry RPC after error@970081062) -> (continuation of Deferred@1596590589 after retry RPC after error@104441340), errback=(continuation of Deferred@937601918 after retry RPC after error@900958221) -> (continuation of Deferred@199420220 after retry RPC after error@370431256) -> (continuation of Deferred@881833471 after retry RPC after error@492150915) -> (continuation of Deferred@31588636 after retry RPC after error@904538285) -> (continuation of Deferred@2088981410 after retry RPC after error@1281533193) -> (continuation of Deferred@1570350968 after retry RPC after error@970081062) -> (continuation of Deferred@1596590589 after retry RPC after error@104441340)))
03:03:52.775 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1155212814(state=PENDING, result=null, callback=(continuation of Deferred@937601918 after retry RPC after error@900958221) -> (continuation of Deferred@199420220 after retry RPC after error@370431256) -> (continuation of Deferred@881833471 after retry RPC after error@492150915) -> (continuation of Deferred@31588636 after retry RPC after error@904538285) -> (continuation of Deferred@2088981410 after retry RPC after error@1281533193) -> (continuation of Deferred@1570350968 after retry RPC after error@970081062) -> (continuation of Deferred@1596590589 after retry RPC after error@104441340), errback=(continuation of Deferred@937601918 after retry RPC after error@900958221) -> (continuation of Deferred@199420220 after retry RPC after error@370431256) -> (continuation of Deferred@881833471 after retry RPC after error@492150915) -> (continuation of Deferred@31588636 after retry RPC after error@904538285) -> (continuation of Deferred@2088981410 after retry RPC after error@1281533193) -> (continuation of Deferred@1570350968 after retry RPC after error@970081062) -> (continuation of Deferred@1596590589 after retry RPC after error@104441340)))
03:03:52.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.853 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.855 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.873 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.954 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@576235273(state=PENDING, result=null, callback=(continuation of Deferred@465219145 after retry RPC after error@759049502) -> (continuation of Deferred@1431717435 after retry RPC after error@844223351) -> (continuation of Deferred@2086136722 after retry RPC after error@542994395) -> (continuation of Deferred@1197314677 after retry RPC after error@2074819458) -> (continuation of Deferred@946029598 after retry RPC after error@1938540414) -> (continuation of Deferred@1492008256 after retry RPC after error@2059107060), errback=(continuation of Deferred@465219145 after retry RPC after error@759049502) -> (continuation of Deferred@1431717435 after retry RPC after error@844223351) -> (continuation of Deferred@2086136722 after retry RPC after error@542994395) -> (continuation of Deferred@1197314677 after retry RPC after error@2074819458) -> (continuation of Deferred@946029598 after retry RPC after error@1938540414) -> (continuation of Deferred@1492008256 after retry RPC after error@2059107060)))
03:03:52.954 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@576235273(state=PENDING, result=null, callback=(continuation of Deferred@465219145 after retry RPC after error@759049502) -> (continuation of Deferred@1431717435 after retry RPC after error@844223351) -> (continuation of Deferred@2086136722 after retry RPC after error@542994395) -> (continuation of Deferred@1197314677 after retry RPC after error@2074819458) -> (continuation of Deferred@946029598 after retry RPC after error@1938540414) -> (continuation of Deferred@1492008256 after retry RPC after error@2059107060), errback=(continuation of Deferred@465219145 after retry RPC after error@759049502) -> (continuation of Deferred@1431717435 after retry RPC after error@844223351) -> (continuation of Deferred@2086136722 after retry RPC after error@542994395) -> (continuation of Deferred@1197314677 after retry RPC after error@2074819458) -> (continuation of Deferred@946029598 after retry RPC after error@1938540414) -> (continuation of Deferred@1492008256 after retry RPC after error@2059107060)))
03:03:52.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:52.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:52.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:52.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:52.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:52.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.193 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@386606934(state=PENDING, result=null, callback=(continuation of Deferred@1143354162 after retry RPC after error@1883789326) -> (continuation of Deferred@904414931 after retry RPC after error@2073556655) -> (continuation of Deferred@249427276 after retry RPC after error@1021793457) -> (continuation of Deferred@1271808800 after retry RPC after error@262727288) -> (continuation of Deferred@1849558356 after retry RPC after error@1761249022) -> (continuation of Deferred@1360273282 after retry RPC after error@432515326) -> (continuation of Deferred@130882103 after retry RPC after error@1002217798), errback=(continuation of Deferred@1143354162 after retry RPC after error@1883789326) -> (continuation of Deferred@904414931 after retry RPC after error@2073556655) -> (continuation of Deferred@249427276 after retry RPC after error@1021793457) -> (continuation of Deferred@1271808800 after retry RPC after error@262727288) -> (continuation of Deferred@1849558356 after retry RPC after error@1761249022) -> (continuation of Deferred@1360273282 after retry RPC after error@432515326) -> (continuation of Deferred@130882103 after retry RPC after error@1002217798)))
03:03:53.194 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@386606934(state=PENDING, result=null, callback=(continuation of Deferred@1143354162 after retry RPC after error@1883789326) -> (continuation of Deferred@904414931 after retry RPC after error@2073556655) -> (continuation of Deferred@249427276 after retry RPC after error@1021793457) -> (continuation of Deferred@1271808800 after retry RPC after error@262727288) -> (continuation of Deferred@1849558356 after retry RPC after error@1761249022) -> (continuation of Deferred@1360273282 after retry RPC after error@432515326) -> (continuation of Deferred@130882103 after retry RPC after error@1002217798), errback=(continuation of Deferred@1143354162 after retry RPC after error@1883789326) -> (continuation of Deferred@904414931 after retry RPC after error@2073556655) -> (continuation of Deferred@249427276 after retry RPC after error@1021793457) -> (continuation of Deferred@1271808800 after retry RPC after error@262727288) -> (continuation of Deferred@1849558356 after retry RPC after error@1761249022) -> (continuation of Deferred@1360273282 after retry RPC after error@432515326) -> (continuation of Deferred@130882103 after retry RPC after error@1002217798)))
03:03:53.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.472 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@519160608(state=PENDING, result=null, callback=(continuation of Deferred@2068390524 after retry RPC after error@380468173) -> (continuation of Deferred@1045259957 after retry RPC after error@783637151) -> (continuation of Deferred@475912183 after retry RPC after error@1632875682) -> (continuation of Deferred@889065203 after retry RPC after error@815520593) -> (continuation of Deferred@1116422362 after retry RPC after error@1048648333) -> (continuation of Deferred@2051805752 after retry RPC after error@342040435) -> (continuation of Deferred@217452725 after retry RPC after error@974464724), errback=(continuation of Deferred@2068390524 after retry RPC after error@380468173) -> (continuation of Deferred@1045259957 after retry RPC after error@783637151) -> (continuation of Deferred@475912183 after retry RPC after error@1632875682) -> (continuation of Deferred@889065203 after retry RPC after error@815520593) -> (continuation of Deferred@1116422362 after retry RPC after error@1048648333) -> (continuation of Deferred@2051805752 after retry RPC after error@342040435) -> (continuation of Deferred@217452725 after retry RPC after error@974464724)))
03:03:53.472 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:53.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.654 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1613571282(state=PENDING, result=null, callback=(continuation of Deferred@1988755957 after retry RPC after error@1481493846) -> (continuation of Deferred@1606017736 after retry RPC after error@1446725209) -> (continuation of Deferred@1880645826 after retry RPC after error@1175004394) -> (continuation of Deferred@1454780737 after retry RPC after error@1513335872) -> (continuation of Deferred@1763625396 after retry RPC after error@1723833865) -> (continuation of Deferred@31754150 after retry RPC after error@1181903478) -> (continuation of Deferred@1505646693 after retry RPC after error@1139530381), errback=(continuation of Deferred@1988755957 after retry RPC after error@1481493846) -> (continuation of Deferred@1606017736 after retry RPC after error@1446725209) -> (continuation of Deferred@1880645826 after retry RPC after error@1175004394) -> (continuation of Deferred@1454780737 after retry RPC after error@1513335872) -> (continuation of Deferred@1763625396 after retry RPC after error@1723833865) -> (continuation of Deferred@31754150 after retry RPC after error@1181903478) -> (continuation of Deferred@1505646693 after retry RPC after error@1139530381)))
03:03:53.655 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1613571282(state=PENDING, result=null, callback=(continuation of Deferred@1988755957 after retry RPC after error@1481493846) -> (continuation of Deferred@1606017736 after retry RPC after error@1446725209) -> (continuation of Deferred@1880645826 after retry RPC after error@1175004394) -> (continuation of Deferred@1454780737 after retry RPC after error@1513335872) -> (continuation of Deferred@1763625396 after retry RPC after error@1723833865) -> (continuation of Deferred@31754150 after retry RPC after error@1181903478) -> (continuation of Deferred@1505646693 after retry RPC after error@1139530381), errback=(continuation of Deferred@1988755957 after retry RPC after error@1481493846) -> (continuation of Deferred@1606017736 after retry RPC after error@1446725209) -> (continuation of Deferred@1880645826 after retry RPC after error@1175004394) -> (continuation of Deferred@1454780737 after retry RPC after error@1513335872) -> (continuation of Deferred@1763625396 after retry RPC after error@1723833865) -> (continuation of Deferred@31754150 after retry RPC after error@1181903478) -> (continuation of Deferred@1505646693 after retry RPC after error@1139530381)))
03:03:53.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.695 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.834 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@551522140(state=PENDING, result=null, callback=(continuation of Deferred@1802887373 after retry RPC after error@58298513) -> (continuation of Deferred@2061245885 after retry RPC after error@648399061) -> (continuation of Deferred@1645360964 after retry RPC after error@757124751) -> (continuation of Deferred@923471715 after retry RPC after error@1071680055) -> (continuation of Deferred@1278947992 after retry RPC after error@1732565286) -> (continuation of Deferred@140227258 after retry RPC after error@869347508) -> (continuation of Deferred@652962724 after retry RPC after error@1730616246), errback=(continuation of Deferred@1802887373 after retry RPC after error@58298513) -> (continuation of Deferred@2061245885 after retry RPC after error@648399061) -> (continuation of Deferred@1645360964 after retry RPC after error@757124751) -> (continuation of Deferred@923471715 after retry RPC after error@1071680055) -> (continuation of Deferred@1278947992 after retry RPC after error@1732565286) -> (continuation of Deferred@140227258 after retry RPC after error@869347508) -> (continuation of Deferred@652962724 after retry RPC after error@1730616246)))
03:03:53.834 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@551522140(state=PENDING, result=null, callback=(continuation of Deferred@1802887373 after retry RPC after error@58298513) -> (continuation of Deferred@2061245885 after retry RPC after error@648399061) -> (continuation of Deferred@1645360964 after retry RPC after error@757124751) -> (continuation of Deferred@923471715 after retry RPC after error@1071680055) -> (continuation of Deferred@1278947992 after retry RPC after error@1732565286) -> (continuation of Deferred@140227258 after retry RPC after error@869347508) -> (continuation of Deferred@652962724 after retry RPC after error@1730616246), errback=(continuation of Deferred@1802887373 after retry RPC after error@58298513) -> (continuation of Deferred@2061245885 after retry RPC after error@648399061) -> (continuation of Deferred@1645360964 after retry RPC after error@757124751) -> (continuation of Deferred@923471715 after retry RPC after error@1071680055) -> (continuation of Deferred@1278947992 after retry RPC after error@1732565286) -> (continuation of Deferred@140227258 after retry RPC after error@869347508) -> (continuation of Deferred@652962724 after retry RPC after error@1730616246)))
03:03:53.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:53.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:53.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:53.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:53.993316 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:53.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:53.995823 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:53.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:53.997540 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:54.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@58104192(state=PENDING, result=null, callback=(continuation of Deferred@859315259 after retry RPC after error@87978634) -> (continuation of Deferred@1432744647 after retry RPC after error@1651039090) -> (continuation of Deferred@289594048 after retry RPC after error@1730039872) -> (continuation of Deferred@496465326 after retry RPC after error@964730405) -> (continuation of Deferred@1041487631 after retry RPC after error@731520221) -> (continuation of Deferred@226895365 after retry RPC after error@818602022) -> (continuation of Deferred@281141919 after retry RPC after error@1799780526), errback=(continuation of Deferred@859315259 after retry RPC after error@87978634) -> (continuation of Deferred@1432744647 after retry RPC after error@1651039090) -> (continuation of Deferred@289594048 after retry RPC after error@1730039872) -> (continuation of Deferred@496465326 after retry RPC after error@964730405) -> (continuation of Deferred@1041487631 after retry RPC after error@731520221) -> (continuation of Deferred@226895365 after retry RPC after error@818602022) -> (continuation of Deferred@281141919 after retry RPC after error@1799780526)))
03:03:54.076 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@58104192(state=PENDING, result=null, callback=(continuation of Deferred@859315259 after retry RPC after error@87978634) -> (continuation of Deferred@1432744647 after retry RPC after error@1651039090) -> (continuation of Deferred@289594048 after retry RPC after error@1730039872) -> (continuation of Deferred@496465326 after retry RPC after error@964730405) -> (continuation of Deferred@1041487631 after retry RPC after error@731520221) -> (continuation of Deferred@226895365 after retry RPC after error@818602022) -> (continuation of Deferred@281141919 after retry RPC after error@1799780526), errback=(continuation of Deferred@859315259 after retry RPC after error@87978634) -> (continuation of Deferred@1432744647 after retry RPC after error@1651039090) -> (continuation of Deferred@289594048 after retry RPC after error@1730039872) -> (continuation of Deferred@496465326 after retry RPC after error@964730405) -> (continuation of Deferred@1041487631 after retry RPC after error@731520221) -> (continuation of Deferred@226895365 after retry RPC after error@818602022) -> (continuation of Deferred@281141919 after retry RPC after error@1799780526)))
03:03:54.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.234 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1873527509(state=PENDING, result=null, callback=(continuation of Deferred@1477648013 after retry RPC after error@1205031149) -> (continuation of Deferred@824594037 after retry RPC after error@2793454) -> (continuation of Deferred@1074601488 after retry RPC after error@486832836) -> (continuation of Deferred@508742391 after retry RPC after error@1247015740) -> (continuation of Deferred@1652984312 after retry RPC after error@1550458166) -> (continuation of Deferred@1424453950 after retry RPC after error@24683149), errback=(continuation of Deferred@1477648013 after retry RPC after error@1205031149) -> (continuation of Deferred@824594037 after retry RPC after error@2793454) -> (continuation of Deferred@1074601488 after retry RPC after error@486832836) -> (continuation of Deferred@508742391 after retry RPC after error@1247015740) -> (continuation of Deferred@1652984312 after retry RPC after error@1550458166) -> (continuation of Deferred@1424453950 after retry RPC after error@24683149)))
03:03:54.234 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1873527509(state=PENDING, result=null, callback=(continuation of Deferred@1477648013 after retry RPC after error@1205031149) -> (continuation of Deferred@824594037 after retry RPC after error@2793454) -> (continuation of Deferred@1074601488 after retry RPC after error@486832836) -> (continuation of Deferred@508742391 after retry RPC after error@1247015740) -> (continuation of Deferred@1652984312 after retry RPC after error@1550458166) -> (continuation of Deferred@1424453950 after retry RPC after error@24683149), errback=(continuation of Deferred@1477648013 after retry RPC after error@1205031149) -> (continuation of Deferred@824594037 after retry RPC after error@2793454) -> (continuation of Deferred@1074601488 after retry RPC after error@486832836) -> (continuation of Deferred@508742391 after retry RPC after error@1247015740) -> (continuation of Deferred@1652984312 after retry RPC after error@1550458166) -> (continuation of Deferred@1424453950 after retry RPC after error@24683149)))
03:03:54.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.494 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1320121489(state=PENDING, result=null, callback=(continuation of Deferred@107033106 after retry RPC after error@2102203828) -> (continuation of Deferred@2105480480 after retry RPC after error@1220672994) -> (continuation of Deferred@1195353930 after retry RPC after error@1380645548) -> (continuation of Deferred@1469653968 after retry RPC after error@284228140) -> (continuation of Deferred@709827215 after retry RPC after error@794106608) -> (continuation of Deferred@835464041 after retry RPC after error@653952742) -> (continuation of Deferred@1949526996 after retry RPC after error@1941335352) -> (continuation of Deferred@603031364 after retry RPC after error@1364129898), errback=(continuation of Deferred@107033106 after retry RPC after error@2102203828) -> (continuation of Deferred@2105480480 after retry RPC after error@1220672994) -> (continuation of Deferred@1195353930 after retry RPC after error@1380645548) -> (continuation of Deferred@1469653968 after retry RPC after error@284228140) -> (continuation of Deferred@709827215 after retry RPC after error@794106608) -> (continuation of Deferred@835464041 after retry RPC after error@653952742) -> (continuation of Deferred@1949526996 after retry RPC after error@1941335352) -> (continuation of Deferred@603031364 after retry RPC after error@1364129898)))
03:03:54.495 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1320121489(state=PENDING, result=null, callback=(continuation of Deferred@107033106 after retry RPC after error@2102203828) -> (continuation of Deferred@2105480480 after retry RPC after error@1220672994) -> (continuation of Deferred@1195353930 after retry RPC after error@1380645548) -> (continuation of Deferred@1469653968 after retry RPC after error@284228140) -> (continuation of Deferred@709827215 after retry RPC after error@794106608) -> (continuation of Deferred@835464041 after retry RPC after error@653952742) -> (continuation of Deferred@1949526996 after retry RPC after error@1941335352) -> (continuation of Deferred@603031364 after retry RPC after error@1364129898), errback=(continuation of Deferred@107033106 after retry RPC after error@2102203828) -> (continuation of Deferred@2105480480 after retry RPC after error@1220672994) -> (continuation of Deferred@1195353930 after retry RPC after error@1380645548) -> (continuation of Deferred@1469653968 after retry RPC after error@284228140) -> (continuation of Deferred@709827215 after retry RPC after error@794106608) -> (continuation of Deferred@835464041 after retry RPC after error@653952742) -> (continuation of Deferred@1949526996 after retry RPC after error@1941335352) -> (continuation of Deferred@603031364 after retry RPC after error@1364129898)))
03:03:54.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.771 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@95944777(state=PENDING, result=null, callback=(continuation of Deferred@1859154267 after retry RPC after error@759888647) -> (continuation of Deferred@461620680 after retry RPC after error@1585198265) -> (continuation of Deferred@868994180 after retry RPC after error@1227066169) -> (continuation of Deferred@1284225616 after retry RPC after error@1944946308) -> (continuation of Deferred@514837393 after retry RPC after error@1258348862) -> (continuation of Deferred@1523092276 after retry RPC after error@1158606671) -> (continuation of Deferred@622675772 after retry RPC after error@1767398320), errback=(continuation of Deferred@1859154267 after retry RPC after error@759888647) -> (continuation of Deferred@461620680 after retry RPC after error@1585198265) -> (continuation of Deferred@868994180 after retry RPC after error@1227066169) -> (continuation of Deferred@1284225616 after retry RPC after error@1944946308) -> (continuation of Deferred@514837393 after retry RPC after error@1258348862) -> (continuation of Deferred@1523092276 after retry RPC after error@1158606671) -> (continuation of Deferred@622675772 after retry RPC after error@1767398320)))
03:03:54.772 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:54.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:54.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:54.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:54.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:54.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:54.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.014 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@709357628(state=PENDING, result=null, callback=(continuation of Deferred@1220749429 after retry RPC after error@929641343) -> (continuation of Deferred@381750967 after retry RPC after error@51567171) -> (continuation of Deferred@1388567252 after retry RPC after error@751823160) -> (continuation of Deferred@845545120 after retry RPC after error@657517165) -> (continuation of Deferred@1802076453 after retry RPC after error@943465305) -> (continuation of Deferred@623991296 after retry RPC after error@99794761) -> (continuation of Deferred@582313595 after retry RPC after error@2076884895), errback=(continuation of Deferred@1220749429 after retry RPC after error@929641343) -> (continuation of Deferred@381750967 after retry RPC after error@51567171) -> (continuation of Deferred@1388567252 after retry RPC after error@751823160) -> (continuation of Deferred@845545120 after retry RPC after error@657517165) -> (continuation of Deferred@1802076453 after retry RPC after error@943465305) -> (continuation of Deferred@623991296 after retry RPC after error@99794761) -> (continuation of Deferred@582313595 after retry RPC after error@2076884895)))
03:03:55.014 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@709357628(state=PENDING, result=null, callback=(continuation of Deferred@1220749429 after retry RPC after error@929641343) -> (continuation of Deferred@381750967 after retry RPC after error@51567171) -> (continuation of Deferred@1388567252 after retry RPC after error@751823160) -> (continuation of Deferred@845545120 after retry RPC after error@657517165) -> (continuation of Deferred@1802076453 after retry RPC after error@943465305) -> (continuation of Deferred@623991296 after retry RPC after error@99794761) -> (continuation of Deferred@582313595 after retry RPC after error@2076884895), errback=(continuation of Deferred@1220749429 after retry RPC after error@929641343) -> (continuation of Deferred@381750967 after retry RPC after error@51567171) -> (continuation of Deferred@1388567252 after retry RPC after error@751823160) -> (continuation of Deferred@845545120 after retry RPC after error@657517165) -> (continuation of Deferred@1802076453 after retry RPC after error@943465305) -> (continuation of Deferred@623991296 after retry RPC after error@99794761) -> (continuation of Deferred@582313595 after retry RPC after error@2076884895)))
03:03:55.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.032 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.033 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.274 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@728602587(state=PENDING, result=null, callback=(continuation of Deferred@1179232960 after retry RPC after error@260509651) -> (continuation of Deferred@1209832783 after retry RPC after error@260575448) -> (continuation of Deferred@77378627 after retry RPC after error@861063828) -> (continuation of Deferred@533235548 after retry RPC after error@1342786221) -> (continuation of Deferred@827773507 after retry RPC after error@343259287) -> (continuation of Deferred@1331441442 after retry RPC after error@1427785798) -> (continuation of Deferred@1381588076 after retry RPC after error@1800529840), errback=(continuation of Deferred@1179232960 after retry RPC after error@260509651) -> (continuation of Deferred@1209832783 after retry RPC after error@260575448) -> (continuation of Deferred@77378627 after retry RPC after error@861063828) -> (continuation of Deferred@533235548 after retry RPC after error@1342786221) -> (continuation of Deferred@827773507 after retry RPC after error@343259287) -> (continuation of Deferred@1331441442 after retry RPC after error@1427785798) -> (continuation of Deferred@1381588076 after retry RPC after error@1800529840)))
03:03:55.274 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@728602587(state=PENDING, result=null, callback=(continuation of Deferred@1179232960 after retry RPC after error@260509651) -> (continuation of Deferred@1209832783 after retry RPC after error@260575448) -> (continuation of Deferred@77378627 after retry RPC after error@861063828) -> (continuation of Deferred@533235548 after retry RPC after error@1342786221) -> (continuation of Deferred@827773507 after retry RPC after error@343259287) -> (continuation of Deferred@1331441442 after retry RPC after error@1427785798) -> (continuation of Deferred@1381588076 after retry RPC after error@1800529840), errback=(continuation of Deferred@1179232960 after retry RPC after error@260509651) -> (continuation of Deferred@1209832783 after retry RPC after error@260575448) -> (continuation of Deferred@77378627 after retry RPC after error@861063828) -> (continuation of Deferred@533235548 after retry RPC after error@1342786221) -> (continuation of Deferred@827773507 after retry RPC after error@343259287) -> (continuation of Deferred@1331441442 after retry RPC after error@1427785798) -> (continuation of Deferred@1381588076 after retry RPC after error@1800529840)))
03:03:55.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.474 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1704386097(state=PENDING, result=null, callback=(continuation of Deferred@1455080724 after retry RPC after error@1347900307) -> (continuation of Deferred@839421862 after retry RPC after error@1639753856) -> (continuation of Deferred@1630426054 after retry RPC after error@185556923) -> (continuation of Deferred@2106361027 after retry RPC after error@2045827930) -> (continuation of Deferred@1783830465 after retry RPC after error@458260407) -> (continuation of Deferred@48450178 after retry RPC after error@2109107087) -> (continuation of Deferred@254383847 after retry RPC after error@387775858), errback=(continuation of Deferred@1455080724 after retry RPC after error@1347900307) -> (continuation of Deferred@839421862 after retry RPC after error@1639753856) -> (continuation of Deferred@1630426054 after retry RPC after error@185556923) -> (continuation of Deferred@2106361027 after retry RPC after error@2045827930) -> (continuation of Deferred@1783830465 after retry RPC after error@458260407) -> (continuation of Deferred@48450178 after retry RPC after error@2109107087) -> (continuation of Deferred@254383847 after retry RPC after error@387775858)))
03:03:55.474 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1704386097(state=PENDING, result=null, callback=(continuation of Deferred@1455080724 after retry RPC after error@1347900307) -> (continuation of Deferred@839421862 after retry RPC after error@1639753856) -> (continuation of Deferred@1630426054 after retry RPC after error@185556923) -> (continuation of Deferred@2106361027 after retry RPC after error@2045827930) -> (continuation of Deferred@1783830465 after retry RPC after error@458260407) -> (continuation of Deferred@48450178 after retry RPC after error@2109107087) -> (continuation of Deferred@254383847 after retry RPC after error@387775858), errback=(continuation of Deferred@1455080724 after retry RPC after error@1347900307) -> (continuation of Deferred@839421862 after retry RPC after error@1639753856) -> (continuation of Deferred@1630426054 after retry RPC after error@185556923) -> (continuation of Deferred@2106361027 after retry RPC after error@2045827930) -> (continuation of Deferred@1783830465 after retry RPC after error@458260407) -> (continuation of Deferred@48450178 after retry RPC after error@2109107087) -> (continuation of Deferred@254383847 after retry RPC after error@387775858)))
03:03:55.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.634 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1468982780(state=PENDING, result=null, callback=(continuation of Deferred@956962241 after retry RPC after error@1397268149) -> (continuation of Deferred@168779026 after retry RPC after error@714008746) -> (continuation of Deferred@1136870439 after retry RPC after error@1416218588) -> (continuation of Deferred@612296590 after retry RPC after error@2036891667) -> (continuation of Deferred@550703461 after retry RPC after error@983459204) -> (continuation of Deferred@1771026382 after retry RPC after error@1423624948), errback=(continuation of Deferred@956962241 after retry RPC after error@1397268149) -> (continuation of Deferred@168779026 after retry RPC after error@714008746) -> (continuation of Deferred@1136870439 after retry RPC after error@1416218588) -> (continuation of Deferred@612296590 after retry RPC after error@2036891667) -> (continuation of Deferred@550703461 after retry RPC after error@983459204) -> (continuation of Deferred@1771026382 after retry RPC after error@1423624948)))
03:03:55.635 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1468982780(state=PENDING, result=null, callback=(continuation of Deferred@956962241 after retry RPC after error@1397268149) -> (continuation of Deferred@168779026 after retry RPC after error@714008746) -> (continuation of Deferred@1136870439 after retry RPC after error@1416218588) -> (continuation of Deferred@612296590 after retry RPC after error@2036891667) -> (continuation of Deferred@550703461 after retry RPC after error@983459204) -> (continuation of Deferred@1771026382 after retry RPC after error@1423624948), errback=(continuation of Deferred@956962241 after retry RPC after error@1397268149) -> (continuation of Deferred@168779026 after retry RPC after error@714008746) -> (continuation of Deferred@1136870439 after retry RPC after error@1416218588) -> (continuation of Deferred@612296590 after retry RPC after error@2036891667) -> (continuation of Deferred@550703461 after retry RPC after error@983459204) -> (continuation of Deferred@1771026382 after retry RPC after error@1423624948)))
03:03:55.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.897 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@984265898(state=PENDING, result=null, callback=(continuation of Deferred@36204034 after retry RPC after error@825337889) -> (continuation of Deferred@104258688 after retry RPC after error@1775646472) -> (continuation of Deferred@684657742 after retry RPC after error@344181206) -> (continuation of Deferred@618779930 after retry RPC after error@2094516473) -> (continuation of Deferred@749344337 after retry RPC after error@653113195) -> (continuation of Deferred@210698153 after retry RPC after error@1323069121) -> (continuation of Deferred@842592725 after retry RPC after error@2032487958), errback=(continuation of Deferred@36204034 after retry RPC after error@825337889) -> (continuation of Deferred@104258688 after retry RPC after error@1775646472) -> (continuation of Deferred@684657742 after retry RPC after error@344181206) -> (continuation of Deferred@618779930 after retry RPC after error@2094516473) -> (continuation of Deferred@749344337 after retry RPC after error@653113195) -> (continuation of Deferred@210698153 after retry RPC after error@1323069121) -> (continuation of Deferred@842592725 after retry RPC after error@2032487958)))
03:03:55.898 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@984265898(state=PENDING, result=null, callback=(continuation of Deferred@36204034 after retry RPC after error@825337889) -> (continuation of Deferred@104258688 after retry RPC after error@1775646472) -> (continuation of Deferred@684657742 after retry RPC after error@344181206) -> (continuation of Deferred@618779930 after retry RPC after error@2094516473) -> (continuation of Deferred@749344337 after retry RPC after error@653113195) -> (continuation of Deferred@210698153 after retry RPC after error@1323069121) -> (continuation of Deferred@842592725 after retry RPC after error@2032487958), errback=(continuation of Deferred@36204034 after retry RPC after error@825337889) -> (continuation of Deferred@104258688 after retry RPC after error@1775646472) -> (continuation of Deferred@684657742 after retry RPC after error@344181206) -> (continuation of Deferred@618779930 after retry RPC after error@2094516473) -> (continuation of Deferred@749344337 after retry RPC after error@653113195) -> (continuation of Deferred@210698153 after retry RPC after error@1323069121) -> (continuation of Deferred@842592725 after retry RPC after error@2032487958)))
03:03:55.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.954 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:55.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:55.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:55.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:55.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:55.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:56.101161 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:03:56.101356 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:56.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.114 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@955989323(state=PENDING, result=null, callback=(continuation of Deferred@2126991836 after retry RPC after error@1190014098) -> (continuation of Deferred@1221061266 after retry RPC after error@699893668) -> (continuation of Deferred@1832545023 after retry RPC after error@1518817654) -> (continuation of Deferred@769897422 after retry RPC after error@947686257) -> (continuation of Deferred@112157459 after retry RPC after error@1724091985) -> (continuation of Deferred@1443773572 after retry RPC after error@175440439) -> (continuation of Deferred@2009030545 after retry RPC after error@225769009), errback=(continuation of Deferred@2126991836 after retry RPC after error@1190014098) -> (continuation of Deferred@1221061266 after retry RPC after error@699893668) -> (continuation of Deferred@1832545023 after retry RPC after error@1518817654) -> (continuation of Deferred@769897422 after retry RPC after error@947686257) -> (continuation of Deferred@112157459 after retry RPC after error@1724091985) -> (continuation of Deferred@1443773572 after retry RPC after error@175440439) -> (continuation of Deferred@2009030545 after retry RPC after error@225769009)))
03:03:56.115 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@955989323(state=PENDING, result=null, callback=(continuation of Deferred@2126991836 after retry RPC after error@1190014098) -> (continuation of Deferred@1221061266 after retry RPC after error@699893668) -> (continuation of Deferred@1832545023 after retry RPC after error@1518817654) -> (continuation of Deferred@769897422 after retry RPC after error@947686257) -> (continuation of Deferred@112157459 after retry RPC after error@1724091985) -> (continuation of Deferred@1443773572 after retry RPC after error@175440439) -> (continuation of Deferred@2009030545 after retry RPC after error@225769009), errback=(continuation of Deferred@2126991836 after retry RPC after error@1190014098) -> (continuation of Deferred@1221061266 after retry RPC after error@699893668) -> (continuation of Deferred@1832545023 after retry RPC after error@1518817654) -> (continuation of Deferred@769897422 after retry RPC after error@947686257) -> (continuation of Deferred@112157459 after retry RPC after error@1724091985) -> (continuation of Deferred@1443773572 after retry RPC after error@175440439) -> (continuation of Deferred@2009030545 after retry RPC after error@225769009)))
03:03:56.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.333 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1414961221(state=PENDING, result=null, callback=(continuation of Deferred@1487724238 after retry RPC after error@157416657) -> (continuation of Deferred@1975287262 after retry RPC after error@365447068) -> (continuation of Deferred@732737861 after retry RPC after error@766330773) -> (continuation of Deferred@1005161843 after retry RPC after error@1816840501) -> (continuation of Deferred@572681056 after retry RPC after error@1702101342) -> (continuation of Deferred@302094998 after retry RPC after error@683207298) -> (continuation of Deferred@426835358 after retry RPC after error@1556832543), errback=(continuation of Deferred@1487724238 after retry RPC after error@157416657) -> (continuation of Deferred@1975287262 after retry RPC after error@365447068) -> (continuation of Deferred@732737861 after retry RPC after error@766330773) -> (continuation of Deferred@1005161843 after retry RPC after error@1816840501) -> (continuation of Deferred@572681056 after retry RPC after error@1702101342) -> (continuation of Deferred@302094998 after retry RPC after error@683207298) -> (continuation of Deferred@426835358 after retry RPC after error@1556832543)))
03:03:56.334 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1414961221(state=PENDING, result=null, callback=(continuation of Deferred@1487724238 after retry RPC after error@157416657) -> (continuation of Deferred@1975287262 after retry RPC after error@365447068) -> (continuation of Deferred@732737861 after retry RPC after error@766330773) -> (continuation of Deferred@1005161843 after retry RPC after error@1816840501) -> (continuation of Deferred@572681056 after retry RPC after error@1702101342) -> (continuation of Deferred@302094998 after retry RPC after error@683207298) -> (continuation of Deferred@426835358 after retry RPC after error@1556832543), errback=(continuation of Deferred@1487724238 after retry RPC after error@157416657) -> (continuation of Deferred@1975287262 after retry RPC after error@365447068) -> (continuation of Deferred@732737861 after retry RPC after error@766330773) -> (continuation of Deferred@1005161843 after retry RPC after error@1816840501) -> (continuation of Deferred@572681056 after retry RPC after error@1702101342) -> (continuation of Deferred@302094998 after retry RPC after error@683207298) -> (continuation of Deferred@426835358 after retry RPC after error@1556832543)))
03:03:56.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.515 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@816762111(state=PENDING, result=null, callback=(continuation of Deferred@288015787 after retry RPC after error@812126552) -> (continuation of Deferred@530177163 after retry RPC after error@1494626995) -> (continuation of Deferred@508998567 after retry RPC after error@1849988068) -> (continuation of Deferred@1063806755 after retry RPC after error@1468221911) -> (continuation of Deferred@2086863627 after retry RPC after error@921855184) -> (continuation of Deferred@1256148050 after retry RPC after error@123697455), errback=(continuation of Deferred@288015787 after retry RPC after error@812126552) -> (continuation of Deferred@530177163 after retry RPC after error@1494626995) -> (continuation of Deferred@508998567 after retry RPC after error@1849988068) -> (continuation of Deferred@1063806755 after retry RPC after error@1468221911) -> (continuation of Deferred@2086863627 after retry RPC after error@921855184) -> (continuation of Deferred@1256148050 after retry RPC after error@123697455)))
03:03:56.515 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@816762111(state=PENDING, result=null, callback=(continuation of Deferred@288015787 after retry RPC after error@812126552) -> (continuation of Deferred@530177163 after retry RPC after error@1494626995) -> (continuation of Deferred@508998567 after retry RPC after error@1849988068) -> (continuation of Deferred@1063806755 after retry RPC after error@1468221911) -> (continuation of Deferred@2086863627 after retry RPC after error@921855184) -> (continuation of Deferred@1256148050 after retry RPC after error@123697455), errback=(continuation of Deferred@288015787 after retry RPC after error@812126552) -> (continuation of Deferred@530177163 after retry RPC after error@1494626995) -> (continuation of Deferred@508998567 after retry RPC after error@1849988068) -> (continuation of Deferred@1063806755 after retry RPC after error@1468221911) -> (continuation of Deferred@2086863627 after retry RPC after error@921855184) -> (continuation of Deferred@1256148050 after retry RPC after error@123697455)))
03:03:56.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.535 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.535 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.535 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.536 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.694 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [5ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1376496598(state=PENDING, result=null, callback=(continuation of Deferred@1169903863 after retry RPC after error@370307295) -> (continuation of Deferred@1343846409 after retry RPC after error@1606626503) -> (continuation of Deferred@1088484128 after retry RPC after error@1165092715) -> (continuation of Deferred@1588791036 after retry RPC after error@407944441) -> (continuation of Deferred@1601058080 after retry RPC after error@367971097) -> (continuation of Deferred@1403829182 after retry RPC after error@1317433926) -> (continuation of Deferred@1617926531 after retry RPC after error@54145172), errback=(continuation of Deferred@1169903863 after retry RPC after error@370307295) -> (continuation of Deferred@1343846409 after retry RPC after error@1606626503) -> (continuation of Deferred@1088484128 after retry RPC after error@1165092715) -> (continuation of Deferred@1588791036 after retry RPC after error@407944441) -> (continuation of Deferred@1601058080 after retry RPC after error@367971097) -> (continuation of Deferred@1403829182 after retry RPC after error@1317433926) -> (continuation of Deferred@1617926531 after retry RPC after error@54145172)))
03:03:56.695 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [5ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1376496598(state=PENDING, result=null, callback=(continuation of Deferred@1169903863 after retry RPC after error@370307295) -> (continuation of Deferred@1343846409 after retry RPC after error@1606626503) -> (continuation of Deferred@1088484128 after retry RPC after error@1165092715) -> (continuation of Deferred@1588791036 after retry RPC after error@407944441) -> (continuation of Deferred@1601058080 after retry RPC after error@367971097) -> (continuation of Deferred@1403829182 after retry RPC after error@1317433926) -> (continuation of Deferred@1617926531 after retry RPC after error@54145172), errback=(continuation of Deferred@1169903863 after retry RPC after error@370307295) -> (continuation of Deferred@1343846409 after retry RPC after error@1606626503) -> (continuation of Deferred@1088484128 after retry RPC after error@1165092715) -> (continuation of Deferred@1588791036 after retry RPC after error@407944441) -> (continuation of Deferred@1601058080 after retry RPC after error@367971097) -> (continuation of Deferred@1403829182 after retry RPC after error@1317433926) -> (continuation of Deferred@1617926531 after retry RPC after error@54145172)))
03:03:56.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.913 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.914 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@188230105(state=PENDING, result=null, callback=(continuation of Deferred@1332976028 after retry RPC after error@1858134716) -> (continuation of Deferred@1282410513 after retry RPC after error@105682599) -> (continuation of Deferred@1428181131 after retry RPC after error@1008508305) -> (continuation of Deferred@1893739139 after retry RPC after error@310076310) -> (continuation of Deferred@1158208858 after retry RPC after error@488648831) -> (continuation of Deferred@1818600810 after retry RPC after error@625679523) -> (continuation of Deferred@648738653 after retry RPC after error@982882367) -> (continuation of Deferred@2073703666 after retry RPC after error@105480486), errback=(continuation of Deferred@1332976028 after retry RPC after error@1858134716) -> (continuation of Deferred@1282410513 after retry RPC after error@105682599) -> (continuation of Deferred@1428181131 after retry RPC after error@1008508305) -> (continuation of Deferred@1893739139 after retry RPC after error@310076310) -> (continuation of Deferred@1158208858 after retry RPC after error@488648831) -> (continuation of Deferred@1818600810 after retry RPC after error@625679523) -> (continuation of Deferred@648738653 after retry RPC after error@982882367) -> (continuation of Deferred@2073703666 after retry RPC after error@105480486)))
03:03:56.915 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@188230105(state=PENDING, result=null, callback=(continuation of Deferred@1332976028 after retry RPC after error@1858134716) -> (continuation of Deferred@1282410513 after retry RPC after error@105682599) -> (continuation of Deferred@1428181131 after retry RPC after error@1008508305) -> (continuation of Deferred@1893739139 after retry RPC after error@310076310) -> (continuation of Deferred@1158208858 after retry RPC after error@488648831) -> (continuation of Deferred@1818600810 after retry RPC after error@625679523) -> (continuation of Deferred@648738653 after retry RPC after error@982882367) -> (continuation of Deferred@2073703666 after retry RPC after error@105480486), errback=(continuation of Deferred@1332976028 after retry RPC after error@1858134716) -> (continuation of Deferred@1282410513 after retry RPC after error@105682599) -> (continuation of Deferred@1428181131 after retry RPC after error@1008508305) -> (continuation of Deferred@1893739139 after retry RPC after error@310076310) -> (continuation of Deferred@1158208858 after retry RPC after error@488648831) -> (continuation of Deferred@1818600810 after retry RPC after error@625679523) -> (continuation of Deferred@648738653 after retry RPC after error@982882367) -> (continuation of Deferred@2073703666 after retry RPC after error@105480486)))
03:03:56.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:56.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:56.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:56.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:56.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:56.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.113 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2134318451(state=PENDING, result=null, callback=(continuation of Deferred@29886473 after retry RPC after error@1962463263) -> (continuation of Deferred@708366082 after retry RPC after error@1839436723) -> (continuation of Deferred@745166414 after retry RPC after error@513803252) -> (continuation of Deferred@1913425917 after retry RPC after error@54768644) -> (continuation of Deferred@2112178692 after retry RPC after error@1673812463) -> (continuation of Deferred@1973506317 after retry RPC after error@1801205801) -> (continuation of Deferred@1019148844 after retry RPC after error@2104805237), errback=(continuation of Deferred@29886473 after retry RPC after error@1962463263) -> (continuation of Deferred@708366082 after retry RPC after error@1839436723) -> (continuation of Deferred@745166414 after retry RPC after error@513803252) -> (continuation of Deferred@1913425917 after retry RPC after error@54768644) -> (continuation of Deferred@2112178692 after retry RPC after error@1673812463) -> (continuation of Deferred@1973506317 after retry RPC after error@1801205801) -> (continuation of Deferred@1019148844 after retry RPC after error@2104805237)))
03:03:57.114 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2134318451(state=PENDING, result=null, callback=(continuation of Deferred@29886473 after retry RPC after error@1962463263) -> (continuation of Deferred@708366082 after retry RPC after error@1839436723) -> (continuation of Deferred@745166414 after retry RPC after error@513803252) -> (continuation of Deferred@1913425917 after retry RPC after error@54768644) -> (continuation of Deferred@2112178692 after retry RPC after error@1673812463) -> (continuation of Deferred@1973506317 after retry RPC after error@1801205801) -> (continuation of Deferred@1019148844 after retry RPC after error@2104805237), errback=(continuation of Deferred@29886473 after retry RPC after error@1962463263) -> (continuation of Deferred@708366082 after retry RPC after error@1839436723) -> (continuation of Deferred@745166414 after retry RPC after error@513803252) -> (continuation of Deferred@1913425917 after retry RPC after error@54768644) -> (continuation of Deferred@2112178692 after retry RPC after error@1673812463) -> (continuation of Deferred@1973506317 after retry RPC after error@1801205801) -> (continuation of Deferred@1019148844 after retry RPC after error@2104805237)))
03:03:57.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.354 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2052125750(state=PENDING, result=null, callback=(continuation of Deferred@32979749 after retry RPC after error@1065330609) -> (continuation of Deferred@1163093075 after retry RPC after error@945805707) -> (continuation of Deferred@4846518 after retry RPC after error@1120505377) -> (continuation of Deferred@476449751 after retry RPC after error@1521683025) -> (continuation of Deferred@363366130 after retry RPC after error@454998752) -> (continuation of Deferred@931803879 after retry RPC after error@1965639621) -> (continuation of Deferred@619171972 after retry RPC after error@1254143810), errback=(continuation of Deferred@32979749 after retry RPC after error@1065330609) -> (continuation of Deferred@1163093075 after retry RPC after error@945805707) -> (continuation of Deferred@4846518 after retry RPC after error@1120505377) -> (continuation of Deferred@476449751 after retry RPC after error@1521683025) -> (continuation of Deferred@363366130 after retry RPC after error@454998752) -> (continuation of Deferred@931803879 after retry RPC after error@1965639621) -> (continuation of Deferred@619171972 after retry RPC after error@1254143810)))
03:03:57.354 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2052125750(state=PENDING, result=null, callback=(continuation of Deferred@32979749 after retry RPC after error@1065330609) -> (continuation of Deferred@1163093075 after retry RPC after error@945805707) -> (continuation of Deferred@4846518 after retry RPC after error@1120505377) -> (continuation of Deferred@476449751 after retry RPC after error@1521683025) -> (continuation of Deferred@363366130 after retry RPC after error@454998752) -> (continuation of Deferred@931803879 after retry RPC after error@1965639621) -> (continuation of Deferred@619171972 after retry RPC after error@1254143810), errback=(continuation of Deferred@32979749 after retry RPC after error@1065330609) -> (continuation of Deferred@1163093075 after retry RPC after error@945805707) -> (continuation of Deferred@4846518 after retry RPC after error@1120505377) -> (continuation of Deferred@476449751 after retry RPC after error@1521683025) -> (continuation of Deferred@363366130 after retry RPC after error@454998752) -> (continuation of Deferred@931803879 after retry RPC after error@1965639621) -> (continuation of Deferred@619171972 after retry RPC after error@1254143810)))
03:03:57.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.594 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@172777604(state=PENDING, result=null, callback=(continuation of Deferred@450643159 after retry RPC after error@229032501) -> (continuation of Deferred@1908866967 after retry RPC after error@409467243) -> (continuation of Deferred@1625680972 after retry RPC after error@1533619471) -> (continuation of Deferred@487934898 after retry RPC after error@975066586) -> (continuation of Deferred@1968593241 after retry RPC after error@1740968873) -> (continuation of Deferred@1354402872 after retry RPC after error@415413104) -> (continuation of Deferred@1595518368 after retry RPC after error@269399473), errback=(continuation of Deferred@450643159 after retry RPC after error@229032501) -> (continuation of Deferred@1908866967 after retry RPC after error@409467243) -> (continuation of Deferred@1625680972 after retry RPC after error@1533619471) -> (continuation of Deferred@487934898 after retry RPC after error@975066586) -> (continuation of Deferred@1968593241 after retry RPC after error@1740968873) -> (continuation of Deferred@1354402872 after retry RPC after error@415413104) -> (continuation of Deferred@1595518368 after retry RPC after error@269399473)))
03:03:57.594 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@172777604(state=PENDING, result=null, callback=(continuation of Deferred@450643159 after retry RPC after error@229032501) -> (continuation of Deferred@1908866967 after retry RPC after error@409467243) -> (continuation of Deferred@1625680972 after retry RPC after error@1533619471) -> (continuation of Deferred@487934898 after retry RPC after error@975066586) -> (continuation of Deferred@1968593241 after retry RPC after error@1740968873) -> (continuation of Deferred@1354402872 after retry RPC after error@415413104) -> (continuation of Deferred@1595518368 after retry RPC after error@269399473), errback=(continuation of Deferred@450643159 after retry RPC after error@229032501) -> (continuation of Deferred@1908866967 after retry RPC after error@409467243) -> (continuation of Deferred@1625680972 after retry RPC after error@1533619471) -> (continuation of Deferred@487934898 after retry RPC after error@975066586) -> (continuation of Deferred@1968593241 after retry RPC after error@1740968873) -> (continuation of Deferred@1354402872 after retry RPC after error@415413104) -> (continuation of Deferred@1595518368 after retry RPC after error@269399473)))
03:03:57.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.872 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1768139927(state=PENDING, result=null, callback=(continuation of Deferred@711938971 after retry RPC after error@661477149) -> (continuation of Deferred@565936158 after retry RPC after error@1439209386) -> (continuation of Deferred@99345667 after retry RPC after error@1215536924) -> (continuation of Deferred@892685692 after retry RPC after error@609352119) -> (continuation of Deferred@2118697923 after retry RPC after error@714549353) -> (continuation of Deferred@894881514 after retry RPC after error@509679030) -> (continuation of Deferred@646941538 after retry RPC after error@893926330), errback=(continuation of Deferred@711938971 after retry RPC after error@661477149) -> (continuation of Deferred@565936158 after retry RPC after error@1439209386) -> (continuation of Deferred@99345667 after retry RPC after error@1215536924) -> (continuation of Deferred@892685692 after retry RPC after error@609352119) -> (continuation of Deferred@2118697923 after retry RPC after error@714549353) -> (continuation of Deferred@894881514 after retry RPC after error@509679030) -> (continuation of Deferred@646941538 after retry RPC after error@893926330)))
03:03:57.872 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:57.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.893 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.895 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:57.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:57.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:57.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:57.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:57.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.113 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@872247563(state=PENDING, result=null, callback=(continuation of Deferred@729784176 after retry RPC after error@2133128583) -> (continuation of Deferred@946536436 after retry RPC after error@1615312091) -> (continuation of Deferred@924070844 after retry RPC after error@1866756116) -> (continuation of Deferred@4106071 after retry RPC after error@1560423143) -> (continuation of Deferred@1260219176 after retry RPC after error@118847119) -> (continuation of Deferred@1927576869 after retry RPC after error@1048392844) -> (continuation of Deferred@513363008 after retry RPC after error@840228882), errback=(continuation of Deferred@729784176 after retry RPC after error@2133128583) -> (continuation of Deferred@946536436 after retry RPC after error@1615312091) -> (continuation of Deferred@924070844 after retry RPC after error@1866756116) -> (continuation of Deferred@4106071 after retry RPC after error@1560423143) -> (continuation of Deferred@1260219176 after retry RPC after error@118847119) -> (continuation of Deferred@1927576869 after retry RPC after error@1048392844) -> (continuation of Deferred@513363008 after retry RPC after error@840228882)))
03:03:58.114 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@872247563(state=PENDING, result=null, callback=(continuation of Deferred@729784176 after retry RPC after error@2133128583) -> (continuation of Deferred@946536436 after retry RPC after error@1615312091) -> (continuation of Deferred@924070844 after retry RPC after error@1866756116) -> (continuation of Deferred@4106071 after retry RPC after error@1560423143) -> (continuation of Deferred@1260219176 after retry RPC after error@118847119) -> (continuation of Deferred@1927576869 after retry RPC after error@1048392844) -> (continuation of Deferred@513363008 after retry RPC after error@840228882), errback=(continuation of Deferred@729784176 after retry RPC after error@2133128583) -> (continuation of Deferred@946536436 after retry RPC after error@1615312091) -> (continuation of Deferred@924070844 after retry RPC after error@1866756116) -> (continuation of Deferred@4106071 after retry RPC after error@1560423143) -> (continuation of Deferred@1260219176 after retry RPC after error@118847119) -> (continuation of Deferred@1927576869 after retry RPC after error@1048392844) -> (continuation of Deferred@513363008 after retry RPC after error@840228882)))
03:03:58.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.295 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2003681006(state=PENDING, result=null, callback=(continuation of Deferred@1550464515 after retry RPC after error@607798795) -> (continuation of Deferred@57062013 after retry RPC after error@434566705) -> (continuation of Deferred@338759968 after retry RPC after error@1657608115) -> (continuation of Deferred@1418638541 after retry RPC after error@2054653715) -> (continuation of Deferred@1725112684 after retry RPC after error@1823068810) -> (continuation of Deferred@1293953806 after retry RPC after error@195650214), errback=(continuation of Deferred@1550464515 after retry RPC after error@607798795) -> (continuation of Deferred@57062013 after retry RPC after error@434566705) -> (continuation of Deferred@338759968 after retry RPC after error@1657608115) -> (continuation of Deferred@1418638541 after retry RPC after error@2054653715) -> (continuation of Deferred@1725112684 after retry RPC after error@1823068810) -> (continuation of Deferred@1293953806 after retry RPC after error@195650214)))
03:03:58.295 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2003681006(state=PENDING, result=null, callback=(continuation of Deferred@1550464515 after retry RPC after error@607798795) -> (continuation of Deferred@57062013 after retry RPC after error@434566705) -> (continuation of Deferred@338759968 after retry RPC after error@1657608115) -> (continuation of Deferred@1418638541 after retry RPC after error@2054653715) -> (continuation of Deferred@1725112684 after retry RPC after error@1823068810) -> (continuation of Deferred@1293953806 after retry RPC after error@195650214), errback=(continuation of Deferred@1550464515 after retry RPC after error@607798795) -> (continuation of Deferred@57062013 after retry RPC after error@434566705) -> (continuation of Deferred@338759968 after retry RPC after error@1657608115) -> (continuation of Deferred@1418638541 after retry RPC after error@2054653715) -> (continuation of Deferred@1725112684 after retry RPC after error@1823068810) -> (continuation of Deferred@1293953806 after retry RPC after error@195650214)))
03:03:58.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.454 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1469706003(state=PENDING, result=null, callback=(continuation of Deferred@84498753 after retry RPC after error@2109650944) -> (continuation of Deferred@1218636422 after retry RPC after error@1348167158) -> (continuation of Deferred@534244845 after retry RPC after error@396441775) -> (continuation of Deferred@1216873991 after retry RPC after error@1140305461) -> (continuation of Deferred@1668258595 after retry RPC after error@2106962849) -> (continuation of Deferred@1965369898 after retry RPC after error@190690493) -> (continuation of Deferred@351567990 after retry RPC after error@1152135666), errback=(continuation of Deferred@84498753 after retry RPC after error@2109650944) -> (continuation of Deferred@1218636422 after retry RPC after error@1348167158) -> (continuation of Deferred@534244845 after retry RPC after error@396441775) -> (continuation of Deferred@1216873991 after retry RPC after error@1140305461) -> (continuation of Deferred@1668258595 after retry RPC after error@2106962849) -> (continuation of Deferred@1965369898 after retry RPC after error@190690493) -> (continuation of Deferred@351567990 after retry RPC after error@1152135666)))
03:03:58.455 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1469706003(state=PENDING, result=null, callback=(continuation of Deferred@84498753 after retry RPC after error@2109650944) -> (continuation of Deferred@1218636422 after retry RPC after error@1348167158) -> (continuation of Deferred@534244845 after retry RPC after error@396441775) -> (continuation of Deferred@1216873991 after retry RPC after error@1140305461) -> (continuation of Deferred@1668258595 after retry RPC after error@2106962849) -> (continuation of Deferred@1965369898 after retry RPC after error@190690493) -> (continuation of Deferred@351567990 after retry RPC after error@1152135666), errback=(continuation of Deferred@84498753 after retry RPC after error@2109650944) -> (continuation of Deferred@1218636422 after retry RPC after error@1348167158) -> (continuation of Deferred@534244845 after retry RPC after error@396441775) -> (continuation of Deferred@1216873991 after retry RPC after error@1140305461) -> (continuation of Deferred@1668258595 after retry RPC after error@2106962849) -> (continuation of Deferred@1965369898 after retry RPC after error@190690493) -> (continuation of Deferred@351567990 after retry RPC after error@1152135666)))
03:03:58.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.694 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@470736441(state=PENDING, result=null, callback=(continuation of Deferred@1074040853 after retry RPC after error@620473626) -> (continuation of Deferred@664619271 after retry RPC after error@1300804827) -> (continuation of Deferred@694681927 after retry RPC after error@1350756877) -> (continuation of Deferred@488260715 after retry RPC after error@356255103) -> (continuation of Deferred@2084002982 after retry RPC after error@1000473519) -> (continuation of Deferred@1951670700 after retry RPC after error@28759460) -> (continuation of Deferred@1201826648 after retry RPC after error@1815525240), errback=(continuation of Deferred@1074040853 after retry RPC after error@620473626) -> (continuation of Deferred@664619271 after retry RPC after error@1300804827) -> (continuation of Deferred@694681927 after retry RPC after error@1350756877) -> (continuation of Deferred@488260715 after retry RPC after error@356255103) -> (continuation of Deferred@2084002982 after retry RPC after error@1000473519) -> (continuation of Deferred@1951670700 after retry RPC after error@28759460) -> (continuation of Deferred@1201826648 after retry RPC after error@1815525240)))
03:03:58.695 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@470736441(state=PENDING, result=null, callback=(continuation of Deferred@1074040853 after retry RPC after error@620473626) -> (continuation of Deferred@664619271 after retry RPC after error@1300804827) -> (continuation of Deferred@694681927 after retry RPC after error@1350756877) -> (continuation of Deferred@488260715 after retry RPC after error@356255103) -> (continuation of Deferred@2084002982 after retry RPC after error@1000473519) -> (continuation of Deferred@1951670700 after retry RPC after error@28759460) -> (continuation of Deferred@1201826648 after retry RPC after error@1815525240), errback=(continuation of Deferred@1074040853 after retry RPC after error@620473626) -> (continuation of Deferred@664619271 after retry RPC after error@1300804827) -> (continuation of Deferred@694681927 after retry RPC after error@1350756877) -> (continuation of Deferred@488260715 after retry RPC after error@356255103) -> (continuation of Deferred@2084002982 after retry RPC after error@1000473519) -> (continuation of Deferred@1951670700 after retry RPC after error@28759460) -> (continuation of Deferred@1201826648 after retry RPC after error@1815525240)))
03:03:58.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.934 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@639669456(state=PENDING, result=null, callback=(continuation of Deferred@1546597240 after retry RPC after error@921789950) -> (continuation of Deferred@1225007895 after retry RPC after error@1805691123) -> (continuation of Deferred@1311896076 after retry RPC after error@1934794913) -> (continuation of Deferred@463874375 after retry RPC after error@1974463984) -> (continuation of Deferred@877646043 after retry RPC after error@1392388443) -> (continuation of Deferred@2025377362 after retry RPC after error@1920101592) -> (continuation of Deferred@955360069 after retry RPC after error@459727345) -> (continuation of Deferred@621966993 after retry RPC after error@1156045383), errback=(continuation of Deferred@1546597240 after retry RPC after error@921789950) -> (continuation of Deferred@1225007895 after retry RPC after error@1805691123) -> (continuation of Deferred@1311896076 after retry RPC after error@1934794913) -> (continuation of Deferred@463874375 after retry RPC after error@1974463984) -> (continuation of Deferred@877646043 after retry RPC after error@1392388443) -> (continuation of Deferred@2025377362 after retry RPC after error@1920101592) -> (continuation of Deferred@955360069 after retry RPC after error@459727345) -> (continuation of Deferred@621966993 after retry RPC after error@1156045383)))
03:03:58.934 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@639669456(state=PENDING, result=null, callback=(continuation of Deferred@1546597240 after retry RPC after error@921789950) -> (continuation of Deferred@1225007895 after retry RPC after error@1805691123) -> (continuation of Deferred@1311896076 after retry RPC after error@1934794913) -> (continuation of Deferred@463874375 after retry RPC after error@1974463984) -> (continuation of Deferred@877646043 after retry RPC after error@1392388443) -> (continuation of Deferred@2025377362 after retry RPC after error@1920101592) -> (continuation of Deferred@955360069 after retry RPC after error@459727345) -> (continuation of Deferred@621966993 after retry RPC after error@1156045383), errback=(continuation of Deferred@1546597240 after retry RPC after error@921789950) -> (continuation of Deferred@1225007895 after retry RPC after error@1805691123) -> (continuation of Deferred@1311896076 after retry RPC after error@1934794913) -> (continuation of Deferred@463874375 after retry RPC after error@1974463984) -> (continuation of Deferred@877646043 after retry RPC after error@1392388443) -> (continuation of Deferred@2025377362 after retry RPC after error@1920101592) -> (continuation of Deferred@955360069 after retry RPC after error@459727345) -> (continuation of Deferred@621966993 after retry RPC after error@1156045383)))
03:03:58.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:58.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:58.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:58.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:58.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:58.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.034 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:59.071715 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:59.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:59.073139 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:03:59.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:03:59.074676 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:03:59.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.193 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@419313190(state=PENDING, result=null, callback=(continuation of Deferred@1062006653 after retry RPC after error@1741447575) -> (continuation of Deferred@1047238148 after retry RPC after error@1464715497) -> (continuation of Deferred@253944442 after retry RPC after error@199682768) -> (continuation of Deferred@1776260638 after retry RPC after error@1349259596) -> (continuation of Deferred@1220872201 after retry RPC after error@1959386027) -> (continuation of Deferred@1630784586 after retry RPC after error@1365595985) -> (continuation of Deferred@830507060 after retry RPC after error@187566380) -> (continuation of Deferred@1954596255 after retry RPC after error@262462337), errback=(continuation of Deferred@1062006653 after retry RPC after error@1741447575) -> (continuation of Deferred@1047238148 after retry RPC after error@1464715497) -> (continuation of Deferred@253944442 after retry RPC after error@199682768) -> (continuation of Deferred@1776260638 after retry RPC after error@1349259596) -> (continuation of Deferred@1220872201 after retry RPC after error@1959386027) -> (continuation of Deferred@1630784586 after retry RPC after error@1365595985) -> (continuation of Deferred@830507060 after retry RPC after error@187566380) -> (continuation of Deferred@1954596255 after retry RPC after error@262462337)))
03:03:59.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@419313190(state=PENDING, result=null, callback=(continuation of Deferred@1062006653 after retry RPC after error@1741447575) -> (continuation of Deferred@1047238148 after retry RPC after error@1464715497) -> (continuation of Deferred@253944442 after retry RPC after error@199682768) -> (continuation of Deferred@1776260638 after retry RPC after error@1349259596) -> (continuation of Deferred@1220872201 after retry RPC after error@1959386027) -> (continuation of Deferred@1630784586 after retry RPC after error@1365595985) -> (continuation of Deferred@830507060 after retry RPC after error@187566380) -> (continuation of Deferred@1954596255 after retry RPC after error@262462337), errback=(continuation of Deferred@1062006653 after retry RPC after error@1741447575) -> (continuation of Deferred@1047238148 after retry RPC after error@1464715497) -> (continuation of Deferred@253944442 after retry RPC after error@199682768) -> (continuation of Deferred@1776260638 after retry RPC after error@1349259596) -> (continuation of Deferred@1220872201 after retry RPC after error@1959386027) -> (continuation of Deferred@1630784586 after retry RPC after error@1365595985) -> (continuation of Deferred@830507060 after retry RPC after error@187566380) -> (continuation of Deferred@1954596255 after retry RPC after error@262462337)))
03:03:59.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.471 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1752923750(state=PENDING, result=null, callback=(continuation of Deferred@951732904 after retry RPC after error@587067203) -> (continuation of Deferred@1023028899 after retry RPC after error@255983380) -> (continuation of Deferred@1698958174 after retry RPC after error@680425282) -> (continuation of Deferred@1164757958 after retry RPC after error@1163207968) -> (continuation of Deferred@972847 after retry RPC after error@1533123339) -> (continuation of Deferred@2020624730 after retry RPC after error@341839697) -> (continuation of Deferred@1650029138 after retry RPC after error@892798161) -> (continuation of Deferred@1333244340 after retry RPC after error@1543058828), errback=(continuation of Deferred@951732904 after retry RPC after error@587067203) -> (continuation of Deferred@1023028899 after retry RPC after error@255983380) -> (continuation of Deferred@1698958174 after retry RPC after error@680425282) -> (continuation of Deferred@1164757958 after retry RPC after error@1163207968) -> (continuation of Deferred@972847 after retry RPC after error@1533123339) -> (continuation of Deferred@2020624730 after retry RPC after error@341839697) -> (continuation of Deferred@1650029138 after retry RPC after error@892798161) -> (continuation of Deferred@1333244340 after retry RPC after error@1543058828)))
03:03:59.472 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:03:59.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.571 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.713 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@820862017(state=PENDING, result=null, callback=(continuation of Deferred@1273532403 after retry RPC after error@1972171256) -> (continuation of Deferred@222462626 after retry RPC after error@1469564959) -> (continuation of Deferred@1477394953 after retry RPC after error@1251900353) -> (continuation of Deferred@1397794722 after retry RPC after error@980306663) -> (continuation of Deferred@408111273 after retry RPC after error@556427281) -> (continuation of Deferred@2006033953 after retry RPC after error@1063314246) -> (continuation of Deferred@1031070211 after retry RPC after error@1340899164), errback=(continuation of Deferred@1273532403 after retry RPC after error@1972171256) -> (continuation of Deferred@222462626 after retry RPC after error@1469564959) -> (continuation of Deferred@1477394953 after retry RPC after error@1251900353) -> (continuation of Deferred@1397794722 after retry RPC after error@980306663) -> (continuation of Deferred@408111273 after retry RPC after error@556427281) -> (continuation of Deferred@2006033953 after retry RPC after error@1063314246) -> (continuation of Deferred@1031070211 after retry RPC after error@1340899164)))
03:03:59.714 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@820862017(state=PENDING, result=null, callback=(continuation of Deferred@1273532403 after retry RPC after error@1972171256) -> (continuation of Deferred@222462626 after retry RPC after error@1469564959) -> (continuation of Deferred@1477394953 after retry RPC after error@1251900353) -> (continuation of Deferred@1397794722 after retry RPC after error@980306663) -> (continuation of Deferred@408111273 after retry RPC after error@556427281) -> (continuation of Deferred@2006033953 after retry RPC after error@1063314246) -> (continuation of Deferred@1031070211 after retry RPC after error@1340899164), errback=(continuation of Deferred@1273532403 after retry RPC after error@1972171256) -> (continuation of Deferred@222462626 after retry RPC after error@1469564959) -> (continuation of Deferred@1477394953 after retry RPC after error@1251900353) -> (continuation of Deferred@1397794722 after retry RPC after error@980306663) -> (continuation of Deferred@408111273 after retry RPC after error@556427281) -> (continuation of Deferred@2006033953 after retry RPC after error@1063314246) -> (continuation of Deferred@1031070211 after retry RPC after error@1340899164)))
03:03:59.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:03:59.973 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2011219284(state=PENDING, result=null, callback=(continuation of Deferred@195108405 after retry RPC after error@1953113402) -> (continuation of Deferred@613673569 after retry RPC after error@1428705730) -> (continuation of Deferred@1450946020 after retry RPC after error@1939669476) -> (continuation of Deferred@1214655764 after retry RPC after error@1599587227) -> (continuation of Deferred@1414021400 after retry RPC after error@1329719380) -> (continuation of Deferred@909533733 after retry RPC after error@1461466211) -> (continuation of Deferred@1103822617 after retry RPC after error@25799740), errback=(continuation of Deferred@195108405 after retry RPC after error@1953113402) -> (continuation of Deferred@613673569 after retry RPC after error@1428705730) -> (continuation of Deferred@1450946020 after retry RPC after error@1939669476) -> (continuation of Deferred@1214655764 after retry RPC after error@1599587227) -> (continuation of Deferred@1414021400 after retry RPC after error@1329719380) -> (continuation of Deferred@909533733 after retry RPC after error@1461466211) -> (continuation of Deferred@1103822617 after retry RPC after error@25799740)))
03:03:59.974 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2011219284(state=PENDING, result=null, callback=(continuation of Deferred@195108405 after retry RPC after error@1953113402) -> (continuation of Deferred@613673569 after retry RPC after error@1428705730) -> (continuation of Deferred@1450946020 after retry RPC after error@1939669476) -> (continuation of Deferred@1214655764 after retry RPC after error@1599587227) -> (continuation of Deferred@1414021400 after retry RPC after error@1329719380) -> (continuation of Deferred@909533733 after retry RPC after error@1461466211) -> (continuation of Deferred@1103822617 after retry RPC after error@25799740), errback=(continuation of Deferred@195108405 after retry RPC after error@1953113402) -> (continuation of Deferred@613673569 after retry RPC after error@1428705730) -> (continuation of Deferred@1450946020 after retry RPC after error@1939669476) -> (continuation of Deferred@1214655764 after retry RPC after error@1599587227) -> (continuation of Deferred@1414021400 after retry RPC after error@1329719380) -> (continuation of Deferred@909533733 after retry RPC after error@1461466211) -> (continuation of Deferred@1103822617 after retry RPC after error@25799740)))
03:03:59.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:03:59.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:03:59.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:03:59.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:03:59.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.015 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.055 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.214 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1505993430(state=PENDING, result=null, callback=(continuation of Deferred@933506911 after retry RPC after error@1708913366) -> (continuation of Deferred@1674514217 after retry RPC after error@660526228) -> (continuation of Deferred@823577079 after retry RPC after error@199344193) -> (continuation of Deferred@281813075 after retry RPC after error@1097313450) -> (continuation of Deferred@1186565120 after retry RPC after error@1138736990) -> (continuation of Deferred@845201870 after retry RPC after error@1310603732) -> (continuation of Deferred@1154311579 after retry RPC after error@2052458259), errback=(continuation of Deferred@933506911 after retry RPC after error@1708913366) -> (continuation of Deferred@1674514217 after retry RPC after error@660526228) -> (continuation of Deferred@823577079 after retry RPC after error@199344193) -> (continuation of Deferred@281813075 after retry RPC after error@1097313450) -> (continuation of Deferred@1186565120 after retry RPC after error@1138736990) -> (continuation of Deferred@845201870 after retry RPC after error@1310603732) -> (continuation of Deferred@1154311579 after retry RPC after error@2052458259)))
03:04:00.215 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1505993430(state=PENDING, result=null, callback=(continuation of Deferred@933506911 after retry RPC after error@1708913366) -> (continuation of Deferred@1674514217 after retry RPC after error@660526228) -> (continuation of Deferred@823577079 after retry RPC after error@199344193) -> (continuation of Deferred@281813075 after retry RPC after error@1097313450) -> (continuation of Deferred@1186565120 after retry RPC after error@1138736990) -> (continuation of Deferred@845201870 after retry RPC after error@1310603732) -> (continuation of Deferred@1154311579 after retry RPC after error@2052458259), errback=(continuation of Deferred@933506911 after retry RPC after error@1708913366) -> (continuation of Deferred@1674514217 after retry RPC after error@660526228) -> (continuation of Deferred@823577079 after retry RPC after error@199344193) -> (continuation of Deferred@281813075 after retry RPC after error@1097313450) -> (continuation of Deferred@1186565120 after retry RPC after error@1138736990) -> (continuation of Deferred@845201870 after retry RPC after error@1310603732) -> (continuation of Deferred@1154311579 after retry RPC after error@2052458259)))
03:04:00.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.234 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.296 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.298 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.491 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [67ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1325953213(state=PENDING, result=null, callback=(continuation of Deferred@1430555528 after retry RPC after error@1868786818) -> (continuation of Deferred@1336814619 after retry RPC after error@1127162059) -> (continuation of Deferred@843508717 after retry RPC after error@1177912923) -> (continuation of Deferred@2017830002 after retry RPC after error@1121385161) -> (continuation of Deferred@1196062973 after retry RPC after error@757541137) -> (continuation of Deferred@2014609139 after retry RPC after error@224510858) -> (continuation of Deferred@435740491 after retry RPC after error@226693828), errback=(continuation of Deferred@1430555528 after retry RPC after error@1868786818) -> (continuation of Deferred@1336814619 after retry RPC after error@1127162059) -> (continuation of Deferred@843508717 after retry RPC after error@1177912923) -> (continuation of Deferred@2017830002 after retry RPC after error@1121385161) -> (continuation of Deferred@1196062973 after retry RPC after error@757541137) -> (continuation of Deferred@2014609139 after retry RPC after error@224510858) -> (continuation of Deferred@435740491 after retry RPC after error@226693828)))
03:04:00.492 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:00.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.593 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.593 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.595 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.632 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.692 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.693 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.693 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.694 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.772 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@660140504(state=PENDING, result=null, callback=(continuation of Deferred@1883437794 after retry RPC after error@497159880) -> (continuation of Deferred@1662562682 after retry RPC after error@53365488) -> (continuation of Deferred@1692153245 after retry RPC after error@1770365824) -> (continuation of Deferred@1328565419 after retry RPC after error@1043916407) -> (continuation of Deferred@1003314416 after retry RPC after error@2140168618) -> (continuation of Deferred@847790055 after retry RPC after error@1274707103) -> (continuation of Deferred@444699596 after retry RPC after error@1717578759), errback=(continuation of Deferred@1883437794 after retry RPC after error@497159880) -> (continuation of Deferred@1662562682 after retry RPC after error@53365488) -> (continuation of Deferred@1692153245 after retry RPC after error@1770365824) -> (continuation of Deferred@1328565419 after retry RPC after error@1043916407) -> (continuation of Deferred@1003314416 after retry RPC after error@2140168618) -> (continuation of Deferred@847790055 after retry RPC after error@1274707103) -> (continuation of Deferred@444699596 after retry RPC after error@1717578759)))
03:04:00.773 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:00.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.795 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.833 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:00.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:00.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:00.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:00.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:00.994 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1911932066(state=PENDING, result=null, callback=(continuation of Deferred@221115992 after retry RPC after error@1247212028) -> (continuation of Deferred@2014789411 after retry RPC after error@655897687) -> (continuation of Deferred@1172670938 after retry RPC after error@952195467) -> (continuation of Deferred@2045679511 after retry RPC after error@523419249) -> (continuation of Deferred@1987075860 after retry RPC after error@1146051920) -> (continuation of Deferred@1234184289 after retry RPC after error@1130087956), errback=(continuation of Deferred@221115992 after retry RPC after error@1247212028) -> (continuation of Deferred@2014789411 after retry RPC after error@655897687) -> (continuation of Deferred@1172670938 after retry RPC after error@952195467) -> (continuation of Deferred@2045679511 after retry RPC after error@523419249) -> (continuation of Deferred@1987075860 after retry RPC after error@1146051920) -> (continuation of Deferred@1234184289 after retry RPC after error@1130087956)))
03:04:00.994 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1911932066(state=PENDING, result=null, callback=(continuation of Deferred@221115992 after retry RPC after error@1247212028) -> (continuation of Deferred@2014789411 after retry RPC after error@655897687) -> (continuation of Deferred@1172670938 after retry RPC after error@952195467) -> (continuation of Deferred@2045679511 after retry RPC after error@523419249) -> (continuation of Deferred@1987075860 after retry RPC after error@1146051920) -> (continuation of Deferred@1234184289 after retry RPC after error@1130087956), errback=(continuation of Deferred@221115992 after retry RPC after error@1247212028) -> (continuation of Deferred@2014789411 after retry RPC after error@655897687) -> (continuation of Deferred@1172670938 after retry RPC after error@952195467) -> (continuation of Deferred@2045679511 after retry RPC after error@523419249) -> (continuation of Deferred@1987075860 after retry RPC after error@1146051920) -> (continuation of Deferred@1234184289 after retry RPC after error@1130087956)))
03:04:01.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:01.103042 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:04:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:01.103271 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:04:01.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.194 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@761749388(state=PENDING, result=null, callback=(continuation of Deferred@2104611853 after retry RPC after error@1144316094) -> (continuation of Deferred@915628065 after retry RPC after error@564122176) -> (continuation of Deferred@1379777474 after retry RPC after error@829394987) -> (continuation of Deferred@483421263 after retry RPC after error@1124249671) -> (continuation of Deferred@2086592590 after retry RPC after error@838715686) -> (continuation of Deferred@785410370 after retry RPC after error@2022077527) -> (continuation of Deferred@1934735257 after retry RPC after error@742057004) -> (continuation of Deferred@26018448 after retry RPC after error@1256385082), errback=(continuation of Deferred@2104611853 after retry RPC after error@1144316094) -> (continuation of Deferred@915628065 after retry RPC after error@564122176) -> (continuation of Deferred@1379777474 after retry RPC after error@829394987) -> (continuation of Deferred@483421263 after retry RPC after error@1124249671) -> (continuation of Deferred@2086592590 after retry RPC after error@838715686) -> (continuation of Deferred@785410370 after retry RPC after error@2022077527) -> (continuation of Deferred@1934735257 after retry RPC after error@742057004) -> (continuation of Deferred@26018448 after retry RPC after error@1256385082)))
03:04:01.195 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@761749388(state=PENDING, result=null, callback=(continuation of Deferred@2104611853 after retry RPC after error@1144316094) -> (continuation of Deferred@915628065 after retry RPC after error@564122176) -> (continuation of Deferred@1379777474 after retry RPC after error@829394987) -> (continuation of Deferred@483421263 after retry RPC after error@1124249671) -> (continuation of Deferred@2086592590 after retry RPC after error@838715686) -> (continuation of Deferred@785410370 after retry RPC after error@2022077527) -> (continuation of Deferred@1934735257 after retry RPC after error@742057004) -> (continuation of Deferred@26018448 after retry RPC after error@1256385082), errback=(continuation of Deferred@2104611853 after retry RPC after error@1144316094) -> (continuation of Deferred@915628065 after retry RPC after error@564122176) -> (continuation of Deferred@1379777474 after retry RPC after error@829394987) -> (continuation of Deferred@483421263 after retry RPC after error@1124249671) -> (continuation of Deferred@2086592590 after retry RPC after error@838715686) -> (continuation of Deferred@785410370 after retry RPC after error@2022077527) -> (continuation of Deferred@1934735257 after retry RPC after error@742057004) -> (continuation of Deferred@26018448 after retry RPC after error@1256385082)))
03:04:01.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.253 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.471 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@75176238(state=PENDING, result=null, callback=(continuation of Deferred@686756803 after retry RPC after error@985077728) -> (continuation of Deferred@2005733875 after retry RPC after error@736347205) -> (continuation of Deferred@2067966961 after retry RPC after error@2034809710) -> (continuation of Deferred@1716862781 after retry RPC after error@1394679390) -> (continuation of Deferred@1028799208 after retry RPC after error@33388007) -> (continuation of Deferred@1318985266 after retry RPC after error@362162769) -> (continuation of Deferred@960481137 after retry RPC after error@1393020924), errback=(continuation of Deferred@686756803 after retry RPC after error@985077728) -> (continuation of Deferred@2005733875 after retry RPC after error@736347205) -> (continuation of Deferred@2067966961 after retry RPC after error@2034809710) -> (continuation of Deferred@1716862781 after retry RPC after error@1394679390) -> (continuation of Deferred@1028799208 after retry RPC after error@33388007) -> (continuation of Deferred@1318985266 after retry RPC after error@362162769) -> (continuation of Deferred@960481137 after retry RPC after error@1393020924)))
03:04:01.472 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:01.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.513 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.515 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.714 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1606075699(state=PENDING, result=null, callback=(continuation of Deferred@1824837078 after retry RPC after error@426452603) -> (continuation of Deferred@1611105510 after retry RPC after error@1763530376) -> (continuation of Deferred@603819008 after retry RPC after error@1667575546) -> (continuation of Deferred@1067034308 after retry RPC after error@1018371753) -> (continuation of Deferred@1944131763 after retry RPC after error@960101067) -> (continuation of Deferred@1264895150 after retry RPC after error@1862720829) -> (continuation of Deferred@341406958 after retry RPC after error@1656765196), errback=(continuation of Deferred@1824837078 after retry RPC after error@426452603) -> (continuation of Deferred@1611105510 after retry RPC after error@1763530376) -> (continuation of Deferred@603819008 after retry RPC after error@1667575546) -> (continuation of Deferred@1067034308 after retry RPC after error@1018371753) -> (continuation of Deferred@1944131763 after retry RPC after error@960101067) -> (continuation of Deferred@1264895150 after retry RPC after error@1862720829) -> (continuation of Deferred@341406958 after retry RPC after error@1656765196)))
03:04:01.715 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1606075699(state=PENDING, result=null, callback=(continuation of Deferred@1824837078 after retry RPC after error@426452603) -> (continuation of Deferred@1611105510 after retry RPC after error@1763530376) -> (continuation of Deferred@603819008 after retry RPC after error@1667575546) -> (continuation of Deferred@1067034308 after retry RPC after error@1018371753) -> (continuation of Deferred@1944131763 after retry RPC after error@960101067) -> (continuation of Deferred@1264895150 after retry RPC after error@1862720829) -> (continuation of Deferred@341406958 after retry RPC after error@1656765196), errback=(continuation of Deferred@1824837078 after retry RPC after error@426452603) -> (continuation of Deferred@1611105510 after retry RPC after error@1763530376) -> (continuation of Deferred@603819008 after retry RPC after error@1667575546) -> (continuation of Deferred@1067034308 after retry RPC after error@1018371753) -> (continuation of Deferred@1944131763 after retry RPC after error@960101067) -> (continuation of Deferred@1264895150 after retry RPC after error@1862720829) -> (continuation of Deferred@341406958 after retry RPC after error@1656765196)))
03:04:01.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:01.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:01.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:01.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:01.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:01.992 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@310576180(state=PENDING, result=null, callback=(continuation of Deferred@1103902640 after retry RPC after error@1849027389) -> (continuation of Deferred@1280461882 after retry RPC after error@1190806304) -> (continuation of Deferred@1714130424 after retry RPC after error@663158275) -> (continuation of Deferred@2037077333 after retry RPC after error@1830611483) -> (continuation of Deferred@1474489869 after retry RPC after error@1199858708) -> (continuation of Deferred@1692811340 after retry RPC after error@1847603961) -> (continuation of Deferred@779238326 after retry RPC after error@1244815016), errback=(continuation of Deferred@1103902640 after retry RPC after error@1849027389) -> (continuation of Deferred@1280461882 after retry RPC after error@1190806304) -> (continuation of Deferred@1714130424 after retry RPC after error@663158275) -> (continuation of Deferred@2037077333 after retry RPC after error@1830611483) -> (continuation of Deferred@1474489869 after retry RPC after error@1199858708) -> (continuation of Deferred@1692811340 after retry RPC after error@1847603961) -> (continuation of Deferred@779238326 after retry RPC after error@1244815016)))
03:04:01.992 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:02.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.254 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1524096348(state=PENDING, result=null, callback=(continuation of Deferred@170485273 after retry RPC after error@2078965394) -> (continuation of Deferred@859594987 after retry RPC after error@1466263277) -> (continuation of Deferred@381541302 after retry RPC after error@1040922909) -> (continuation of Deferred@1860189455 after retry RPC after error@315522096) -> (continuation of Deferred@1792435636 after retry RPC after error@225602082) -> (continuation of Deferred@1653946054 after retry RPC after error@56355160) -> (continuation of Deferred@1526318914 after retry RPC after error@1153729861) -> (continuation of Deferred@254588957 after retry RPC after error@1398899081), errback=(continuation of Deferred@170485273 after retry RPC after error@2078965394) -> (continuation of Deferred@859594987 after retry RPC after error@1466263277) -> (continuation of Deferred@381541302 after retry RPC after error@1040922909) -> (continuation of Deferred@1860189455 after retry RPC after error@315522096) -> (continuation of Deferred@1792435636 after retry RPC after error@225602082) -> (continuation of Deferred@1653946054 after retry RPC after error@56355160) -> (continuation of Deferred@1526318914 after retry RPC after error@1153729861) -> (continuation of Deferred@254588957 after retry RPC after error@1398899081)))
03:04:02.255 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1524096348(state=PENDING, result=null, callback=(continuation of Deferred@170485273 after retry RPC after error@2078965394) -> (continuation of Deferred@859594987 after retry RPC after error@1466263277) -> (continuation of Deferred@381541302 after retry RPC after error@1040922909) -> (continuation of Deferred@1860189455 after retry RPC after error@315522096) -> (continuation of Deferred@1792435636 after retry RPC after error@225602082) -> (continuation of Deferred@1653946054 after retry RPC after error@56355160) -> (continuation of Deferred@1526318914 after retry RPC after error@1153729861) -> (continuation of Deferred@254588957 after retry RPC after error@1398899081), errback=(continuation of Deferred@170485273 after retry RPC after error@2078965394) -> (continuation of Deferred@859594987 after retry RPC after error@1466263277) -> (continuation of Deferred@381541302 after retry RPC after error@1040922909) -> (continuation of Deferred@1860189455 after retry RPC after error@315522096) -> (continuation of Deferred@1792435636 after retry RPC after error@225602082) -> (continuation of Deferred@1653946054 after retry RPC after error@56355160) -> (continuation of Deferred@1526318914 after retry RPC after error@1153729861) -> (continuation of Deferred@254588957 after retry RPC after error@1398899081)))
03:04:02.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.454 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1923385117(state=PENDING, result=null, callback=(continuation of Deferred@1341383334 after retry RPC after error@58586605) -> (continuation of Deferred@849169929 after retry RPC after error@1683471144) -> (continuation of Deferred@887194582 after retry RPC after error@1482672698) -> (continuation of Deferred@6957630 after retry RPC after error@1497190368) -> (continuation of Deferred@1643808802 after retry RPC after error@1196593213) -> (continuation of Deferred@381184307 after retry RPC after error@625567307), errback=(continuation of Deferred@1341383334 after retry RPC after error@58586605) -> (continuation of Deferred@849169929 after retry RPC after error@1683471144) -> (continuation of Deferred@887194582 after retry RPC after error@1482672698) -> (continuation of Deferred@6957630 after retry RPC after error@1497190368) -> (continuation of Deferred@1643808802 after retry RPC after error@1196593213) -> (continuation of Deferred@381184307 after retry RPC after error@625567307)))
03:04:02.454 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1923385117(state=PENDING, result=null, callback=(continuation of Deferred@1341383334 after retry RPC after error@58586605) -> (continuation of Deferred@849169929 after retry RPC after error@1683471144) -> (continuation of Deferred@887194582 after retry RPC after error@1482672698) -> (continuation of Deferred@6957630 after retry RPC after error@1497190368) -> (continuation of Deferred@1643808802 after retry RPC after error@1196593213) -> (continuation of Deferred@381184307 after retry RPC after error@625567307), errback=(continuation of Deferred@1341383334 after retry RPC after error@58586605) -> (continuation of Deferred@849169929 after retry RPC after error@1683471144) -> (continuation of Deferred@887194582 after retry RPC after error@1482672698) -> (continuation of Deferred@6957630 after retry RPC after error@1497190368) -> (continuation of Deferred@1643808802 after retry RPC after error@1196593213) -> (continuation of Deferred@381184307 after retry RPC after error@625567307)))
03:04:02.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.633 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@249147796(state=PENDING, result=null, callback=(continuation of Deferred@1408772652 after retry RPC after error@1543740823) -> (continuation of Deferred@110897740 after retry RPC after error@1166700058) -> (continuation of Deferred@1462863309 after retry RPC after error@387859865) -> (continuation of Deferred@1882706812 after retry RPC after error@1725667806) -> (continuation of Deferred@1025468197 after retry RPC after error@1481166738) -> (continuation of Deferred@371870676 after retry RPC after error@746927526), errback=(continuation of Deferred@1408772652 after retry RPC after error@1543740823) -> (continuation of Deferred@110897740 after retry RPC after error@1166700058) -> (continuation of Deferred@1462863309 after retry RPC after error@387859865) -> (continuation of Deferred@1882706812 after retry RPC after error@1725667806) -> (continuation of Deferred@1025468197 after retry RPC after error@1481166738) -> (continuation of Deferred@371870676 after retry RPC after error@746927526)))
03:04:02.634 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@249147796(state=PENDING, result=null, callback=(continuation of Deferred@1408772652 after retry RPC after error@1543740823) -> (continuation of Deferred@110897740 after retry RPC after error@1166700058) -> (continuation of Deferred@1462863309 after retry RPC after error@387859865) -> (continuation of Deferred@1882706812 after retry RPC after error@1725667806) -> (continuation of Deferred@1025468197 after retry RPC after error@1481166738) -> (continuation of Deferred@371870676 after retry RPC after error@746927526), errback=(continuation of Deferred@1408772652 after retry RPC after error@1543740823) -> (continuation of Deferred@110897740 after retry RPC after error@1166700058) -> (continuation of Deferred@1462863309 after retry RPC after error@387859865) -> (continuation of Deferred@1882706812 after retry RPC after error@1725667806) -> (continuation of Deferred@1025468197 after retry RPC after error@1481166738) -> (continuation of Deferred@371870676 after retry RPC after error@746927526)))
03:04:02.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.853 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@667486962(state=PENDING, result=null, callback=(continuation of Deferred@49298785 after retry RPC after error@1306784195) -> (continuation of Deferred@2100723469 after retry RPC after error@315057347) -> (continuation of Deferred@299849935 after retry RPC after error@1138806494) -> (continuation of Deferred@224399288 after retry RPC after error@606634923) -> (continuation of Deferred@1331347137 after retry RPC after error@168370122) -> (continuation of Deferred@127524092 after retry RPC after error@1741435519), errback=(continuation of Deferred@49298785 after retry RPC after error@1306784195) -> (continuation of Deferred@2100723469 after retry RPC after error@315057347) -> (continuation of Deferred@299849935 after retry RPC after error@1138806494) -> (continuation of Deferred@224399288 after retry RPC after error@606634923) -> (continuation of Deferred@1331347137 after retry RPC after error@168370122) -> (continuation of Deferred@127524092 after retry RPC after error@1741435519)))
03:04:02.854 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@667486962(state=PENDING, result=null, callback=(continuation of Deferred@49298785 after retry RPC after error@1306784195) -> (continuation of Deferred@2100723469 after retry RPC after error@315057347) -> (continuation of Deferred@299849935 after retry RPC after error@1138806494) -> (continuation of Deferred@224399288 after retry RPC after error@606634923) -> (continuation of Deferred@1331347137 after retry RPC after error@168370122) -> (continuation of Deferred@127524092 after retry RPC after error@1741435519), errback=(continuation of Deferred@49298785 after retry RPC after error@1306784195) -> (continuation of Deferred@2100723469 after retry RPC after error@315057347) -> (continuation of Deferred@299849935 after retry RPC after error@1138806494) -> (continuation of Deferred@224399288 after retry RPC after error@606634923) -> (continuation of Deferred@1331347137 after retry RPC after error@168370122) -> (continuation of Deferred@127524092 after retry RPC after error@1741435519)))
03:04:02.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:02.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:02.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:02.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:02.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:02.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.033 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@462673247(state=PENDING, result=null, callback=(continuation of Deferred@114957144 after retry RPC after error@1049016486) -> (continuation of Deferred@2137466682 after retry RPC after error@2052795556) -> (continuation of Deferred@739108972 after retry RPC after error@658847427) -> (continuation of Deferred@1641909222 after retry RPC after error@1119925621) -> (continuation of Deferred@21855177 after retry RPC after error@254850573) -> (continuation of Deferred@37719734 after retry RPC after error@1158932633), errback=(continuation of Deferred@114957144 after retry RPC after error@1049016486) -> (continuation of Deferred@2137466682 after retry RPC after error@2052795556) -> (continuation of Deferred@739108972 after retry RPC after error@658847427) -> (continuation of Deferred@1641909222 after retry RPC after error@1119925621) -> (continuation of Deferred@21855177 after retry RPC after error@254850573) -> (continuation of Deferred@37719734 after retry RPC after error@1158932633)))
03:04:03.033 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@462673247(state=PENDING, result=null, callback=(continuation of Deferred@114957144 after retry RPC after error@1049016486) -> (continuation of Deferred@2137466682 after retry RPC after error@2052795556) -> (continuation of Deferred@739108972 after retry RPC after error@658847427) -> (continuation of Deferred@1641909222 after retry RPC after error@1119925621) -> (continuation of Deferred@21855177 after retry RPC after error@254850573) -> (continuation of Deferred@37719734 after retry RPC after error@1158932633), errback=(continuation of Deferred@114957144 after retry RPC after error@1049016486) -> (continuation of Deferred@2137466682 after retry RPC after error@2052795556) -> (continuation of Deferred@739108972 after retry RPC after error@658847427) -> (continuation of Deferred@1641909222 after retry RPC after error@1119925621) -> (continuation of Deferred@21855177 after retry RPC after error@254850573) -> (continuation of Deferred@37719734 after retry RPC after error@1158932633)))
03:04:03.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.114 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.253 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.253 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.255 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1479709635(state=PENDING, result=null, callback=(continuation of Deferred@1651997386 after retry RPC after error@1707348273) -> (continuation of Deferred@1859665586 after retry RPC after error@582485025) -> (continuation of Deferred@2067827606 after retry RPC after error@1044926550) -> (continuation of Deferred@290116739 after retry RPC after error@1932608391) -> (continuation of Deferred@206731852 after retry RPC after error@1894007152) -> (continuation of Deferred@927825187 after retry RPC after error@1310276918) -> (continuation of Deferred@1367785123 after retry RPC after error@62622242) -> (continuation of Deferred@1606634259 after retry RPC after error@1479814480), errback=(continuation of Deferred@1651997386 after retry RPC after error@1707348273) -> (continuation of Deferred@1859665586 after retry RPC after error@582485025) -> (continuation of Deferred@2067827606 after retry RPC after error@1044926550) -> (continuation of Deferred@290116739 after retry RPC after error@1932608391) -> (continuation of Deferred@206731852 after retry RPC after error@1894007152) -> (continuation of Deferred@927825187 after retry RPC after error@1310276918) -> (continuation of Deferred@1367785123 after retry RPC after error@62622242) -> (continuation of Deferred@1606634259 after retry RPC after error@1479814480)))
03:04:03.255 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1479709635(state=PENDING, result=null, callback=(continuation of Deferred@1651997386 after retry RPC after error@1707348273) -> (continuation of Deferred@1859665586 after retry RPC after error@582485025) -> (continuation of Deferred@2067827606 after retry RPC after error@1044926550) -> (continuation of Deferred@290116739 after retry RPC after error@1932608391) -> (continuation of Deferred@206731852 after retry RPC after error@1894007152) -> (continuation of Deferred@927825187 after retry RPC after error@1310276918) -> (continuation of Deferred@1367785123 after retry RPC after error@62622242) -> (continuation of Deferred@1606634259 after retry RPC after error@1479814480), errback=(continuation of Deferred@1651997386 after retry RPC after error@1707348273) -> (continuation of Deferred@1859665586 after retry RPC after error@582485025) -> (continuation of Deferred@2067827606 after retry RPC after error@1044926550) -> (continuation of Deferred@290116739 after retry RPC after error@1932608391) -> (continuation of Deferred@206731852 after retry RPC after error@1894007152) -> (continuation of Deferred@927825187 after retry RPC after error@1310276918) -> (continuation of Deferred@1367785123 after retry RPC after error@62622242) -> (continuation of Deferred@1606634259 after retry RPC after error@1479814480)))
03:04:03.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.413 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@743074667(state=PENDING, result=null, callback=(continuation of Deferred@698170526 after retry RPC after error@1601296509) -> (continuation of Deferred@1230812150 after retry RPC after error@917670021) -> (continuation of Deferred@1980447489 after retry RPC after error@661170061) -> (continuation of Deferred@193422229 after retry RPC after error@701707456) -> (continuation of Deferred@112181704 after retry RPC after error@1687771743) -> (continuation of Deferred@1412564626 after retry RPC after error@1685540320) -> (continuation of Deferred@481190183 after retry RPC after error@843811663), errback=(continuation of Deferred@698170526 after retry RPC after error@1601296509) -> (continuation of Deferred@1230812150 after retry RPC after error@917670021) -> (continuation of Deferred@1980447489 after retry RPC after error@661170061) -> (continuation of Deferred@193422229 after retry RPC after error@701707456) -> (continuation of Deferred@112181704 after retry RPC after error@1687771743) -> (continuation of Deferred@1412564626 after retry RPC after error@1685540320) -> (continuation of Deferred@481190183 after retry RPC after error@843811663)))
03:04:03.414 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@743074667(state=PENDING, result=null, callback=(continuation of Deferred@698170526 after retry RPC after error@1601296509) -> (continuation of Deferred@1230812150 after retry RPC after error@917670021) -> (continuation of Deferred@1980447489 after retry RPC after error@661170061) -> (continuation of Deferred@193422229 after retry RPC after error@701707456) -> (continuation of Deferred@112181704 after retry RPC after error@1687771743) -> (continuation of Deferred@1412564626 after retry RPC after error@1685540320) -> (continuation of Deferred@481190183 after retry RPC after error@843811663), errback=(continuation of Deferred@698170526 after retry RPC after error@1601296509) -> (continuation of Deferred@1230812150 after retry RPC after error@917670021) -> (continuation of Deferred@1980447489 after retry RPC after error@661170061) -> (continuation of Deferred@193422229 after retry RPC after error@701707456) -> (continuation of Deferred@112181704 after retry RPC after error@1687771743) -> (continuation of Deferred@1412564626 after retry RPC after error@1685540320) -> (continuation of Deferred@481190183 after retry RPC after error@843811663)))
03:04:03.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.431 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.634 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1826420434(state=PENDING, result=null, callback=(continuation of Deferred@275042071 after retry RPC after error@747410778) -> (continuation of Deferred@425452490 after retry RPC after error@305851477) -> (continuation of Deferred@605062514 after retry RPC after error@1820254959) -> (continuation of Deferred@721230105 after retry RPC after error@1868566615) -> (continuation of Deferred@1341862876 after retry RPC after error@1409931606) -> (continuation of Deferred@377370399 after retry RPC after error@2046000943) -> (continuation of Deferred@1870710302 after retry RPC after error@1968792017), errback=(continuation of Deferred@275042071 after retry RPC after error@747410778) -> (continuation of Deferred@425452490 after retry RPC after error@305851477) -> (continuation of Deferred@605062514 after retry RPC after error@1820254959) -> (continuation of Deferred@721230105 after retry RPC after error@1868566615) -> (continuation of Deferred@1341862876 after retry RPC after error@1409931606) -> (continuation of Deferred@377370399 after retry RPC after error@2046000943) -> (continuation of Deferred@1870710302 after retry RPC after error@1968792017)))
03:04:03.634 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1826420434(state=PENDING, result=null, callback=(continuation of Deferred@275042071 after retry RPC after error@747410778) -> (continuation of Deferred@425452490 after retry RPC after error@305851477) -> (continuation of Deferred@605062514 after retry RPC after error@1820254959) -> (continuation of Deferred@721230105 after retry RPC after error@1868566615) -> (continuation of Deferred@1341862876 after retry RPC after error@1409931606) -> (continuation of Deferred@377370399 after retry RPC after error@2046000943) -> (continuation of Deferred@1870710302 after retry RPC after error@1968792017), errback=(continuation of Deferred@275042071 after retry RPC after error@747410778) -> (continuation of Deferred@425452490 after retry RPC after error@305851477) -> (continuation of Deferred@605062514 after retry RPC after error@1820254959) -> (continuation of Deferred@721230105 after retry RPC after error@1868566615) -> (continuation of Deferred@1341862876 after retry RPC after error@1409931606) -> (continuation of Deferred@377370399 after retry RPC after error@2046000943) -> (continuation of Deferred@1870710302 after retry RPC after error@1968792017)))
03:04:03.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.854 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1383937610(state=PENDING, result=null, callback=(continuation of Deferred@1830080696 after retry RPC after error@118099254) -> (continuation of Deferred@297411024 after retry RPC after error@709790411) -> (continuation of Deferred@1637092906 after retry RPC after error@916511891) -> (continuation of Deferred@1993658166 after retry RPC after error@680664053) -> (continuation of Deferred@1773581715 after retry RPC after error@1505933426) -> (continuation of Deferred@155148547 after retry RPC after error@826973794) -> (continuation of Deferred@1837936629 after retry RPC after error@566336286), errback=(continuation of Deferred@1830080696 after retry RPC after error@118099254) -> (continuation of Deferred@297411024 after retry RPC after error@709790411) -> (continuation of Deferred@1637092906 after retry RPC after error@916511891) -> (continuation of Deferred@1993658166 after retry RPC after error@680664053) -> (continuation of Deferred@1773581715 after retry RPC after error@1505933426) -> (continuation of Deferred@155148547 after retry RPC after error@826973794) -> (continuation of Deferred@1837936629 after retry RPC after error@566336286)))
03:04:03.854 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1383937610(state=PENDING, result=null, callback=(continuation of Deferred@1830080696 after retry RPC after error@118099254) -> (continuation of Deferred@297411024 after retry RPC after error@709790411) -> (continuation of Deferred@1637092906 after retry RPC after error@916511891) -> (continuation of Deferred@1993658166 after retry RPC after error@680664053) -> (continuation of Deferred@1773581715 after retry RPC after error@1505933426) -> (continuation of Deferred@155148547 after retry RPC after error@826973794) -> (continuation of Deferred@1837936629 after retry RPC after error@566336286), errback=(continuation of Deferred@1830080696 after retry RPC after error@118099254) -> (continuation of Deferred@297411024 after retry RPC after error@709790411) -> (continuation of Deferred@1637092906 after retry RPC after error@916511891) -> (continuation of Deferred@1993658166 after retry RPC after error@680664053) -> (continuation of Deferred@1773581715 after retry RPC after error@1505933426) -> (continuation of Deferred@155148547 after retry RPC after error@826973794) -> (continuation of Deferred@1837936629 after retry RPC after error@566336286)))
03:04:03.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:03.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:03.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:03.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:03.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:03.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.053 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@907183935(state=PENDING, result=null, callback=(continuation of Deferred@1944268722 after retry RPC after error@1818921912) -> (continuation of Deferred@759471591 after retry RPC after error@256760847) -> (continuation of Deferred@1698295019 after retry RPC after error@1572248767) -> (continuation of Deferred@1419020619 after retry RPC after error@903383666) -> (continuation of Deferred@523007638 after retry RPC after error@1981170344) -> (continuation of Deferred@104350704 after retry RPC after error@2009252447) -> (continuation of Deferred@187073271 after retry RPC after error@1340183097), errback=(continuation of Deferred@1944268722 after retry RPC after error@1818921912) -> (continuation of Deferred@759471591 after retry RPC after error@256760847) -> (continuation of Deferred@1698295019 after retry RPC after error@1572248767) -> (continuation of Deferred@1419020619 after retry RPC after error@903383666) -> (continuation of Deferred@523007638 after retry RPC after error@1981170344) -> (continuation of Deferred@104350704 after retry RPC after error@2009252447) -> (continuation of Deferred@187073271 after retry RPC after error@1340183097)))
03:04:04.054 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@907183935(state=PENDING, result=null, callback=(continuation of Deferred@1944268722 after retry RPC after error@1818921912) -> (continuation of Deferred@759471591 after retry RPC after error@256760847) -> (continuation of Deferred@1698295019 after retry RPC after error@1572248767) -> (continuation of Deferred@1419020619 after retry RPC after error@903383666) -> (continuation of Deferred@523007638 after retry RPC after error@1981170344) -> (continuation of Deferred@104350704 after retry RPC after error@2009252447) -> (continuation of Deferred@187073271 after retry RPC after error@1340183097), errback=(continuation of Deferred@1944268722 after retry RPC after error@1818921912) -> (continuation of Deferred@759471591 after retry RPC after error@256760847) -> (continuation of Deferred@1698295019 after retry RPC after error@1572248767) -> (continuation of Deferred@1419020619 after retry RPC after error@903383666) -> (continuation of Deferred@523007638 after retry RPC after error@1981170344) -> (continuation of Deferred@104350704 after retry RPC after error@2009252447) -> (continuation of Deferred@187073271 after retry RPC after error@1340183097)))
03:04:04.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:04.124377 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:04.125888 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:04.127461 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:04:04.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.217 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.217 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.217 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.217 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.218 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=145), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@428742734(state=PENDING, result=null, callback=(continuation of Deferred@457506085 after retry RPC after error@1109089361) -> (continuation of Deferred@1201955744 after retry RPC after error@780226418) -> (continuation of Deferred@303051346 after retry RPC after error@175185301) -> (continuation of Deferred@1856820760 after retry RPC after error@1249046112) -> (continuation of Deferred@1499715550 after retry RPC after error@1543304402) -> (continuation of Deferred@1399387555 after retry RPC after error@238288960), errback=(continuation of Deferred@457506085 after retry RPC after error@1109089361) -> (continuation of Deferred@1201955744 after retry RPC after error@780226418) -> (continuation of Deferred@303051346 after retry RPC after error@175185301) -> (continuation of Deferred@1856820760 after retry RPC after error@1249046112) -> (continuation of Deferred@1499715550 after retry RPC after error@1543304402) -> (continuation of Deferred@1399387555 after retry RPC after error@238288960)))
03:04:04.218 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=145), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@428742734(state=PENDING, result=null, callback=(continuation of Deferred@457506085 after retry RPC after error@1109089361) -> (continuation of Deferred@1201955744 after retry RPC after error@780226418) -> (continuation of Deferred@303051346 after retry RPC after error@175185301) -> (continuation of Deferred@1856820760 after retry RPC after error@1249046112) -> (continuation of Deferred@1499715550 after retry RPC after error@1543304402) -> (continuation of Deferred@1399387555 after retry RPC after error@238288960), errback=(continuation of Deferred@457506085 after retry RPC after error@1109089361) -> (continuation of Deferred@1201955744 after retry RPC after error@780226418) -> (continuation of Deferred@303051346 after retry RPC after error@175185301) -> (continuation of Deferred@1856820760 after retry RPC after error@1249046112) -> (continuation of Deferred@1499715550 after retry RPC after error@1543304402) -> (continuation of Deferred@1399387555 after retry RPC after error@238288960)))
03:04:04.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.374 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@395463308(state=PENDING, result=null, callback=(continuation of Deferred@228911001 after retry RPC after error@2091214144) -> (continuation of Deferred@287980135 after retry RPC after error@342480100) -> (continuation of Deferred@1054568927 after retry RPC after error@1761840161) -> (continuation of Deferred@700082788 after retry RPC after error@1940800835) -> (continuation of Deferred@379492192 after retry RPC after error@1661563670) -> (continuation of Deferred@429575257 after retry RPC after error@404686452), errback=(continuation of Deferred@228911001 after retry RPC after error@2091214144) -> (continuation of Deferred@287980135 after retry RPC after error@342480100) -> (continuation of Deferred@1054568927 after retry RPC after error@1761840161) -> (continuation of Deferred@700082788 after retry RPC after error@1940800835) -> (continuation of Deferred@379492192 after retry RPC after error@1661563670) -> (continuation of Deferred@429575257 after retry RPC after error@404686452)))
03:04:04.375 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@395463308(state=PENDING, result=null, callback=(continuation of Deferred@228911001 after retry RPC after error@2091214144) -> (continuation of Deferred@287980135 after retry RPC after error@342480100) -> (continuation of Deferred@1054568927 after retry RPC after error@1761840161) -> (continuation of Deferred@700082788 after retry RPC after error@1940800835) -> (continuation of Deferred@379492192 after retry RPC after error@1661563670) -> (continuation of Deferred@429575257 after retry RPC after error@404686452), errback=(continuation of Deferred@228911001 after retry RPC after error@2091214144) -> (continuation of Deferred@287980135 after retry RPC after error@342480100) -> (continuation of Deferred@1054568927 after retry RPC after error@1761840161) -> (continuation of Deferred@700082788 after retry RPC after error@1940800835) -> (continuation of Deferred@379492192 after retry RPC after error@1661563670) -> (continuation of Deferred@429575257 after retry RPC after error@404686452)))
03:04:04.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.554 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2028405608(state=PENDING, result=null, callback=(continuation of Deferred@848974754 after retry RPC after error@149219865) -> (continuation of Deferred@1384007935 after retry RPC after error@306380187) -> (continuation of Deferred@1923459890 after retry RPC after error@1572746077) -> (continuation of Deferred@2099749285 after retry RPC after error@2014062015) -> (continuation of Deferred@665845175 after retry RPC after error@195499716) -> (continuation of Deferred@1271768220 after retry RPC after error@1383416199), errback=(continuation of Deferred@848974754 after retry RPC after error@149219865) -> (continuation of Deferred@1384007935 after retry RPC after error@306380187) -> (continuation of Deferred@1923459890 after retry RPC after error@1572746077) -> (continuation of Deferred@2099749285 after retry RPC after error@2014062015) -> (continuation of Deferred@665845175 after retry RPC after error@195499716) -> (continuation of Deferred@1271768220 after retry RPC after error@1383416199)))
03:04:04.554 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2028405608(state=PENDING, result=null, callback=(continuation of Deferred@848974754 after retry RPC after error@149219865) -> (continuation of Deferred@1384007935 after retry RPC after error@306380187) -> (continuation of Deferred@1923459890 after retry RPC after error@1572746077) -> (continuation of Deferred@2099749285 after retry RPC after error@2014062015) -> (continuation of Deferred@665845175 after retry RPC after error@195499716) -> (continuation of Deferred@1271768220 after retry RPC after error@1383416199), errback=(continuation of Deferred@848974754 after retry RPC after error@149219865) -> (continuation of Deferred@1384007935 after retry RPC after error@306380187) -> (continuation of Deferred@1923459890 after retry RPC after error@1572746077) -> (continuation of Deferred@2099749285 after retry RPC after error@2014062015) -> (continuation of Deferred@665845175 after retry RPC after error@195499716) -> (continuation of Deferred@1271768220 after retry RPC after error@1383416199)))
03:04:04.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.774 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2093972064(state=PENDING, result=null, callback=(continuation of Deferred@1562164626 after retry RPC after error@939058490) -> (continuation of Deferred@1708792920 after retry RPC after error@1648457465) -> (continuation of Deferred@1518974734 after retry RPC after error@707283024) -> (continuation of Deferred@530954641 after retry RPC after error@1991963170) -> (continuation of Deferred@1629231676 after retry RPC after error@161395847) -> (continuation of Deferred@1002831124 after retry RPC after error@449782700) -> (continuation of Deferred@461571891 after retry RPC after error@1741009480), errback=(continuation of Deferred@1562164626 after retry RPC after error@939058490) -> (continuation of Deferred@1708792920 after retry RPC after error@1648457465) -> (continuation of Deferred@1518974734 after retry RPC after error@707283024) -> (continuation of Deferred@530954641 after retry RPC after error@1991963170) -> (continuation of Deferred@1629231676 after retry RPC after error@161395847) -> (continuation of Deferred@1002831124 after retry RPC after error@449782700) -> (continuation of Deferred@461571891 after retry RPC after error@1741009480)))
03:04:04.775 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2093972064(state=PENDING, result=null, callback=(continuation of Deferred@1562164626 after retry RPC after error@939058490) -> (continuation of Deferred@1708792920 after retry RPC after error@1648457465) -> (continuation of Deferred@1518974734 after retry RPC after error@707283024) -> (continuation of Deferred@530954641 after retry RPC after error@1991963170) -> (continuation of Deferred@1629231676 after retry RPC after error@161395847) -> (continuation of Deferred@1002831124 after retry RPC after error@449782700) -> (continuation of Deferred@461571891 after retry RPC after error@1741009480), errback=(continuation of Deferred@1562164626 after retry RPC after error@939058490) -> (continuation of Deferred@1708792920 after retry RPC after error@1648457465) -> (continuation of Deferred@1518974734 after retry RPC after error@707283024) -> (continuation of Deferred@530954641 after retry RPC after error@1991963170) -> (continuation of Deferred@1629231676 after retry RPC after error@161395847) -> (continuation of Deferred@1002831124 after retry RPC after error@449782700) -> (continuation of Deferred@461571891 after retry RPC after error@1741009480)))
03:04:04.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:04.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:04.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:04.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:04.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:04.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.051 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1694598794(state=PENDING, result=null, callback=(continuation of Deferred@1894657877 after retry RPC after error@1544014647) -> (continuation of Deferred@1547047471 after retry RPC after error@904588021) -> (continuation of Deferred@880803738 after retry RPC after error@1447285841) -> (continuation of Deferred@981996508 after retry RPC after error@1482156858) -> (continuation of Deferred@320818309 after retry RPC after error@1538712555) -> (continuation of Deferred@567942578 after retry RPC after error@1177762952) -> (continuation of Deferred@790703023 after retry RPC after error@1139031067) -> (continuation of Deferred@849514072 after retry RPC after error@1839866872), errback=(continuation of Deferred@1894657877 after retry RPC after error@1544014647) -> (continuation of Deferred@1547047471 after retry RPC after error@904588021) -> (continuation of Deferred@880803738 after retry RPC after error@1447285841) -> (continuation of Deferred@981996508 after retry RPC after error@1482156858) -> (continuation of Deferred@320818309 after retry RPC after error@1538712555) -> (continuation of Deferred@567942578 after retry RPC after error@1177762952) -> (continuation of Deferred@790703023 after retry RPC after error@1139031067) -> (continuation of Deferred@849514072 after retry RPC after error@1839866872)))
03:04:05.052 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:05.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.112 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.331 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1228291837(state=PENDING, result=null, callback=(continuation of Deferred@222043271 after retry RPC after error@1447569814) -> (continuation of Deferred@1909408552 after retry RPC after error@167135802) -> (continuation of Deferred@1682159527 after retry RPC after error@1934632480) -> (continuation of Deferred@2014182753 after retry RPC after error@1125244864) -> (continuation of Deferred@982951562 after retry RPC after error@1208346334) -> (continuation of Deferred@1601498186 after retry RPC after error@307681211) -> (continuation of Deferred@494637453 after retry RPC after error@2058336234), errback=(continuation of Deferred@222043271 after retry RPC after error@1447569814) -> (continuation of Deferred@1909408552 after retry RPC after error@167135802) -> (continuation of Deferred@1682159527 after retry RPC after error@1934632480) -> (continuation of Deferred@2014182753 after retry RPC after error@1125244864) -> (continuation of Deferred@982951562 after retry RPC after error@1208346334) -> (continuation of Deferred@1601498186 after retry RPC after error@307681211) -> (continuation of Deferred@494637453 after retry RPC after error@2058336234)))
03:04:05.332 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:05.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.391 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.392 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.493 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.534 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1489389522(state=PENDING, result=null, callback=(continuation of Deferred@1070930162 after retry RPC after error@1268629006) -> (continuation of Deferred@1219478786 after retry RPC after error@582869778) -> (continuation of Deferred@892890431 after retry RPC after error@326355032) -> (continuation of Deferred@633510113 after retry RPC after error@1747601976) -> (continuation of Deferred@512590743 after retry RPC after error@322021654) -> (continuation of Deferred@570768897 after retry RPC after error@407161843) -> (continuation of Deferred@1780965975 after retry RPC after error@1958747932), errback=(continuation of Deferred@1070930162 after retry RPC after error@1268629006) -> (continuation of Deferred@1219478786 after retry RPC after error@582869778) -> (continuation of Deferred@892890431 after retry RPC after error@326355032) -> (continuation of Deferred@633510113 after retry RPC after error@1747601976) -> (continuation of Deferred@512590743 after retry RPC after error@322021654) -> (continuation of Deferred@570768897 after retry RPC after error@407161843) -> (continuation of Deferred@1780965975 after retry RPC after error@1958747932)))
03:04:05.534 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1489389522(state=PENDING, result=null, callback=(continuation of Deferred@1070930162 after retry RPC after error@1268629006) -> (continuation of Deferred@1219478786 after retry RPC after error@582869778) -> (continuation of Deferred@892890431 after retry RPC after error@326355032) -> (continuation of Deferred@633510113 after retry RPC after error@1747601976) -> (continuation of Deferred@512590743 after retry RPC after error@322021654) -> (continuation of Deferred@570768897 after retry RPC after error@407161843) -> (continuation of Deferred@1780965975 after retry RPC after error@1958747932), errback=(continuation of Deferred@1070930162 after retry RPC after error@1268629006) -> (continuation of Deferred@1219478786 after retry RPC after error@582869778) -> (continuation of Deferred@892890431 after retry RPC after error@326355032) -> (continuation of Deferred@633510113 after retry RPC after error@1747601976) -> (continuation of Deferred@512590743 after retry RPC after error@322021654) -> (continuation of Deferred@570768897 after retry RPC after error@407161843) -> (continuation of Deferred@1780965975 after retry RPC after error@1958747932)))
03:04:05.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.552 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.573 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.773 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@892882019(state=PENDING, result=null, callback=(continuation of Deferred@94511648 after retry RPC after error@2006875612) -> (continuation of Deferred@161603962 after retry RPC after error@2123235193) -> (continuation of Deferred@1789555114 after retry RPC after error@1851691454) -> (continuation of Deferred@1752307518 after retry RPC after error@2039831422) -> (continuation of Deferred@1082868423 after retry RPC after error@1514334217) -> (continuation of Deferred@577888152 after retry RPC after error@1902919171) -> (continuation of Deferred@1802229651 after retry RPC after error@648308823), errback=(continuation of Deferred@94511648 after retry RPC after error@2006875612) -> (continuation of Deferred@161603962 after retry RPC after error@2123235193) -> (continuation of Deferred@1789555114 after retry RPC after error@1851691454) -> (continuation of Deferred@1752307518 after retry RPC after error@2039831422) -> (continuation of Deferred@1082868423 after retry RPC after error@1514334217) -> (continuation of Deferred@577888152 after retry RPC after error@1902919171) -> (continuation of Deferred@1802229651 after retry RPC after error@648308823)))
03:04:05.774 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@892882019(state=PENDING, result=null, callback=(continuation of Deferred@94511648 after retry RPC after error@2006875612) -> (continuation of Deferred@161603962 after retry RPC after error@2123235193) -> (continuation of Deferred@1789555114 after retry RPC after error@1851691454) -> (continuation of Deferred@1752307518 after retry RPC after error@2039831422) -> (continuation of Deferred@1082868423 after retry RPC after error@1514334217) -> (continuation of Deferred@577888152 after retry RPC after error@1902919171) -> (continuation of Deferred@1802229651 after retry RPC after error@648308823), errback=(continuation of Deferred@94511648 after retry RPC after error@2006875612) -> (continuation of Deferred@161603962 after retry RPC after error@2123235193) -> (continuation of Deferred@1789555114 after retry RPC after error@1851691454) -> (continuation of Deferred@1752307518 after retry RPC after error@2039831422) -> (continuation of Deferred@1082868423 after retry RPC after error@1514334217) -> (continuation of Deferred@577888152 after retry RPC after error@1902919171) -> (continuation of Deferred@1802229651 after retry RPC after error@648308823)))
03:04:05.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:05.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:05.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:05.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:05.914 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:05.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.052 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2103133786(state=PENDING, result=null, callback=(continuation of Deferred@1054980386 after retry RPC after error@1968651936) -> (continuation of Deferred@971601990 after retry RPC after error@514293943) -> (continuation of Deferred@758907624 after retry RPC after error@1641188305) -> (continuation of Deferred@327939301 after retry RPC after error@822836251) -> (continuation of Deferred@2087519120 after retry RPC after error@160732111) -> (continuation of Deferred@1165259460 after retry RPC after error@796210645) -> (continuation of Deferred@999346728 after retry RPC after error@381105278), errback=(continuation of Deferred@1054980386 after retry RPC after error@1968651936) -> (continuation of Deferred@971601990 after retry RPC after error@514293943) -> (continuation of Deferred@758907624 after retry RPC after error@1641188305) -> (continuation of Deferred@327939301 after retry RPC after error@822836251) -> (continuation of Deferred@2087519120 after retry RPC after error@160732111) -> (continuation of Deferred@1165259460 after retry RPC after error@796210645) -> (continuation of Deferred@999346728 after retry RPC after error@381105278)))
03:04:06.052 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:06.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:06.105014 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:04:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:06.105244 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:04:06.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.112 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.151 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.172 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.214 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@449089417(state=PENDING, result=null, callback=(continuation of Deferred@2077055577 after retry RPC after error@830134757) -> (continuation of Deferred@830318470 after retry RPC after error@259776231) -> (continuation of Deferred@511001283 after retry RPC after error@2032604441) -> (continuation of Deferred@632998956 after retry RPC after error@1952952986) -> (continuation of Deferred@1076925839 after retry RPC after error@33480525) -> (continuation of Deferred@1918286066 after retry RPC after error@996947776), errback=(continuation of Deferred@2077055577 after retry RPC after error@830134757) -> (continuation of Deferred@830318470 after retry RPC after error@259776231) -> (continuation of Deferred@511001283 after retry RPC after error@2032604441) -> (continuation of Deferred@632998956 after retry RPC after error@1952952986) -> (continuation of Deferred@1076925839 after retry RPC after error@33480525) -> (continuation of Deferred@1918286066 after retry RPC after error@996947776)))
03:04:06.218 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@449089417(state=PENDING, result=null, callback=(continuation of Deferred@2077055577 after retry RPC after error@830134757) -> (continuation of Deferred@830318470 after retry RPC after error@259776231) -> (continuation of Deferred@511001283 after retry RPC after error@2032604441) -> (continuation of Deferred@632998956 after retry RPC after error@1952952986) -> (continuation of Deferred@1076925839 after retry RPC after error@33480525) -> (continuation of Deferred@1918286066 after retry RPC after error@996947776), errback=(continuation of Deferred@2077055577 after retry RPC after error@830134757) -> (continuation of Deferred@830318470 after retry RPC after error@259776231) -> (continuation of Deferred@511001283 after retry RPC after error@2032604441) -> (continuation of Deferred@632998956 after retry RPC after error@1952952986) -> (continuation of Deferred@1076925839 after retry RPC after error@33480525) -> (continuation of Deferred@1918286066 after retry RPC after error@996947776)))
03:04:06.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.394 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1691229492(state=PENDING, result=null, callback=(continuation of Deferred@2084020189 after retry RPC after error@170317458) -> (continuation of Deferred@1787638511 after retry RPC after error@2118038839) -> (continuation of Deferred@1052181594 after retry RPC after error@35247960) -> (continuation of Deferred@25200574 after retry RPC after error@177999284) -> (continuation of Deferred@210718057 after retry RPC after error@1088481463) -> (continuation of Deferred@1171794530 after retry RPC after error@801337024) -> (continuation of Deferred@1506904660 after retry RPC after error@506684553), errback=(continuation of Deferred@2084020189 after retry RPC after error@170317458) -> (continuation of Deferred@1787638511 after retry RPC after error@2118038839) -> (continuation of Deferred@1052181594 after retry RPC after error@35247960) -> (continuation of Deferred@25200574 after retry RPC after error@177999284) -> (continuation of Deferred@210718057 after retry RPC after error@1088481463) -> (continuation of Deferred@1171794530 after retry RPC after error@801337024) -> (continuation of Deferred@1506904660 after retry RPC after error@506684553)))
03:04:06.395 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1691229492(state=PENDING, result=null, callback=(continuation of Deferred@2084020189 after retry RPC after error@170317458) -> (continuation of Deferred@1787638511 after retry RPC after error@2118038839) -> (continuation of Deferred@1052181594 after retry RPC after error@35247960) -> (continuation of Deferred@25200574 after retry RPC after error@177999284) -> (continuation of Deferred@210718057 after retry RPC after error@1088481463) -> (continuation of Deferred@1171794530 after retry RPC after error@801337024) -> (continuation of Deferred@1506904660 after retry RPC after error@506684553), errback=(continuation of Deferred@2084020189 after retry RPC after error@170317458) -> (continuation of Deferred@1787638511 after retry RPC after error@2118038839) -> (continuation of Deferred@1052181594 after retry RPC after error@35247960) -> (continuation of Deferred@25200574 after retry RPC after error@177999284) -> (continuation of Deferred@210718057 after retry RPC after error@1088481463) -> (continuation of Deferred@1171794530 after retry RPC after error@801337024) -> (continuation of Deferred@1506904660 after retry RPC after error@506684553)))
03:04:06.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.653 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1582962740(state=PENDING, result=null, callback=(continuation of Deferred@1631430708 after retry RPC after error@617636081) -> (continuation of Deferred@2146543361 after retry RPC after error@1910807411) -> (continuation of Deferred@2098520435 after retry RPC after error@1524941256) -> (continuation of Deferred@677362746 after retry RPC after error@2100663617) -> (continuation of Deferred@661712109 after retry RPC after error@1533449034) -> (continuation of Deferred@244140656 after retry RPC after error@1598245245) -> (continuation of Deferred@2138362364 after retry RPC after error@283632027), errback=(continuation of Deferred@1631430708 after retry RPC after error@617636081) -> (continuation of Deferred@2146543361 after retry RPC after error@1910807411) -> (continuation of Deferred@2098520435 after retry RPC after error@1524941256) -> (continuation of Deferred@677362746 after retry RPC after error@2100663617) -> (continuation of Deferred@661712109 after retry RPC after error@1533449034) -> (continuation of Deferred@244140656 after retry RPC after error@1598245245) -> (continuation of Deferred@2138362364 after retry RPC after error@283632027)))
03:04:06.654 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1582962740(state=PENDING, result=null, callback=(continuation of Deferred@1631430708 after retry RPC after error@617636081) -> (continuation of Deferred@2146543361 after retry RPC after error@1910807411) -> (continuation of Deferred@2098520435 after retry RPC after error@1524941256) -> (continuation of Deferred@677362746 after retry RPC after error@2100663617) -> (continuation of Deferred@661712109 after retry RPC after error@1533449034) -> (continuation of Deferred@244140656 after retry RPC after error@1598245245) -> (continuation of Deferred@2138362364 after retry RPC after error@283632027), errback=(continuation of Deferred@1631430708 after retry RPC after error@617636081) -> (continuation of Deferred@2146543361 after retry RPC after error@1910807411) -> (continuation of Deferred@2098520435 after retry RPC after error@1524941256) -> (continuation of Deferred@677362746 after retry RPC after error@2100663617) -> (continuation of Deferred@661712109 after retry RPC after error@1533449034) -> (continuation of Deferred@244140656 after retry RPC after error@1598245245) -> (continuation of Deferred@2138362364 after retry RPC after error@283632027)))
03:04:06.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.733 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.893 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1524362088(state=PENDING, result=null, callback=(continuation of Deferred@288669913 after retry RPC after error@587878392) -> (continuation of Deferred@2109955643 after retry RPC after error@2026269248) -> (continuation of Deferred@1338204041 after retry RPC after error@1044884486) -> (continuation of Deferred@1502744435 after retry RPC after error@307100267) -> (continuation of Deferred@1193791676 after retry RPC after error@1149461442) -> (continuation of Deferred@228787638 after retry RPC after error@1883257035) -> (continuation of Deferred@161195087 after retry RPC after error@1457081293), errback=(continuation of Deferred@288669913 after retry RPC after error@587878392) -> (continuation of Deferred@2109955643 after retry RPC after error@2026269248) -> (continuation of Deferred@1338204041 after retry RPC after error@1044884486) -> (continuation of Deferred@1502744435 after retry RPC after error@307100267) -> (continuation of Deferred@1193791676 after retry RPC after error@1149461442) -> (continuation of Deferred@228787638 after retry RPC after error@1883257035) -> (continuation of Deferred@161195087 after retry RPC after error@1457081293)))
03:04:06.894 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1524362088(state=PENDING, result=null, callback=(continuation of Deferred@288669913 after retry RPC after error@587878392) -> (continuation of Deferred@2109955643 after retry RPC after error@2026269248) -> (continuation of Deferred@1338204041 after retry RPC after error@1044884486) -> (continuation of Deferred@1502744435 after retry RPC after error@307100267) -> (continuation of Deferred@1193791676 after retry RPC after error@1149461442) -> (continuation of Deferred@228787638 after retry RPC after error@1883257035) -> (continuation of Deferred@161195087 after retry RPC after error@1457081293), errback=(continuation of Deferred@288669913 after retry RPC after error@587878392) -> (continuation of Deferred@2109955643 after retry RPC after error@2026269248) -> (continuation of Deferred@1338204041 after retry RPC after error@1044884486) -> (continuation of Deferred@1502744435 after retry RPC after error@307100267) -> (continuation of Deferred@1193791676 after retry RPC after error@1149461442) -> (continuation of Deferred@228787638 after retry RPC after error@1883257035) -> (continuation of Deferred@161195087 after retry RPC after error@1457081293)))
03:04:06.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:06.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:06.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:06.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:06.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:06.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:07.094885 18574 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:04:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:07.095148 18574 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:04:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:07.096086 18574 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:04:07.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:07.097435 18216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:07.098762 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:07.098929 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:07.099051 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:07.099273 18574 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:04:07.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.154 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.154 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1386903362(state=PENDING, result=null, callback=(continuation of Deferred@412611458 after retry RPC after error@844399514) -> (continuation of Deferred@673486505 after retry RPC after error@691401864) -> (continuation of Deferred@236521935 after retry RPC after error@407307205) -> (continuation of Deferred@341467066 after retry RPC after error@315455501) -> (continuation of Deferred@1423727850 after retry RPC after error@1948591263) -> (continuation of Deferred@843321902 after retry RPC after error@1261439355) -> (continuation of Deferred@2109552635 after retry RPC after error@543290639), errback=(continuation of Deferred@412611458 after retry RPC after error@844399514) -> (continuation of Deferred@673486505 after retry RPC after error@691401864) -> (continuation of Deferred@236521935 after retry RPC after error@407307205) -> (continuation of Deferred@341467066 after retry RPC after error@315455501) -> (continuation of Deferred@1423727850 after retry RPC after error@1948591263) -> (continuation of Deferred@843321902 after retry RPC after error@1261439355) -> (continuation of Deferred@2109552635 after retry RPC after error@543290639)))
03:04:07.155 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1386903362(state=PENDING, result=null, callback=(continuation of Deferred@412611458 after retry RPC after error@844399514) -> (continuation of Deferred@673486505 after retry RPC after error@691401864) -> (continuation of Deferred@236521935 after retry RPC after error@407307205) -> (continuation of Deferred@341467066 after retry RPC after error@315455501) -> (continuation of Deferred@1423727850 after retry RPC after error@1948591263) -> (continuation of Deferred@843321902 after retry RPC after error@1261439355) -> (continuation of Deferred@2109552635 after retry RPC after error@543290639), errback=(continuation of Deferred@412611458 after retry RPC after error@844399514) -> (continuation of Deferred@673486505 after retry RPC after error@691401864) -> (continuation of Deferred@236521935 after retry RPC after error@407307205) -> (continuation of Deferred@341467066 after retry RPC after error@315455501) -> (continuation of Deferred@1423727850 after retry RPC after error@1948591263) -> (continuation of Deferred@843321902 after retry RPC after error@1261439355) -> (continuation of Deferred@2109552635 after retry RPC after error@543290639)))
03:04:07.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.353 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@169399262(state=PENDING, result=null, callback=(continuation of Deferred@1786219945 after retry RPC after error@197061645) -> (continuation of Deferred@669949023 after retry RPC after error@1805921631) -> (continuation of Deferred@1030291271 after retry RPC after error@1095414371) -> (continuation of Deferred@311808625 after retry RPC after error@1477565353) -> (continuation of Deferred@578393387 after retry RPC after error@917824173) -> (continuation of Deferred@177125456 after retry RPC after error@1544873531), errback=(continuation of Deferred@1786219945 after retry RPC after error@197061645) -> (continuation of Deferred@669949023 after retry RPC after error@1805921631) -> (continuation of Deferred@1030291271 after retry RPC after error@1095414371) -> (continuation of Deferred@311808625 after retry RPC after error@1477565353) -> (continuation of Deferred@578393387 after retry RPC after error@917824173) -> (continuation of Deferred@177125456 after retry RPC after error@1544873531)))
03:04:07.354 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@169399262(state=PENDING, result=null, callback=(continuation of Deferred@1786219945 after retry RPC after error@197061645) -> (continuation of Deferred@669949023 after retry RPC after error@1805921631) -> (continuation of Deferred@1030291271 after retry RPC after error@1095414371) -> (continuation of Deferred@311808625 after retry RPC after error@1477565353) -> (continuation of Deferred@578393387 after retry RPC after error@917824173) -> (continuation of Deferred@177125456 after retry RPC after error@1544873531), errback=(continuation of Deferred@1786219945 after retry RPC after error@197061645) -> (continuation of Deferred@669949023 after retry RPC after error@1805921631) -> (continuation of Deferred@1030291271 after retry RPC after error@1095414371) -> (continuation of Deferred@311808625 after retry RPC after error@1477565353) -> (continuation of Deferred@578393387 after retry RPC after error@917824173) -> (continuation of Deferred@177125456 after retry RPC after error@1544873531)))
03:04:07.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.594 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@674247962(state=PENDING, result=null, callback=(continuation of Deferred@166312909 after retry RPC after error@224823655) -> (continuation of Deferred@699757076 after retry RPC after error@8657629) -> (continuation of Deferred@1148714707 after retry RPC after error@2103671435) -> (continuation of Deferred@2135200481 after retry RPC after error@1597386022) -> (continuation of Deferred@617402741 after retry RPC after error@1117605957) -> (continuation of Deferred@429199725 after retry RPC after error@2007253092) -> (continuation of Deferred@891379868 after retry RPC after error@496369925), errback=(continuation of Deferred@166312909 after retry RPC after error@224823655) -> (continuation of Deferred@699757076 after retry RPC after error@8657629) -> (continuation of Deferred@1148714707 after retry RPC after error@2103671435) -> (continuation of Deferred@2135200481 after retry RPC after error@1597386022) -> (continuation of Deferred@617402741 after retry RPC after error@1117605957) -> (continuation of Deferred@429199725 after retry RPC after error@2007253092) -> (continuation of Deferred@891379868 after retry RPC after error@496369925)))
03:04:07.594 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@674247962(state=PENDING, result=null, callback=(continuation of Deferred@166312909 after retry RPC after error@224823655) -> (continuation of Deferred@699757076 after retry RPC after error@8657629) -> (continuation of Deferred@1148714707 after retry RPC after error@2103671435) -> (continuation of Deferred@2135200481 after retry RPC after error@1597386022) -> (continuation of Deferred@617402741 after retry RPC after error@1117605957) -> (continuation of Deferred@429199725 after retry RPC after error@2007253092) -> (continuation of Deferred@891379868 after retry RPC after error@496369925), errback=(continuation of Deferred@166312909 after retry RPC after error@224823655) -> (continuation of Deferred@699757076 after retry RPC after error@8657629) -> (continuation of Deferred@1148714707 after retry RPC after error@2103671435) -> (continuation of Deferred@2135200481 after retry RPC after error@1597386022) -> (continuation of Deferred@617402741 after retry RPC after error@1117605957) -> (continuation of Deferred@429199725 after retry RPC after error@2007253092) -> (continuation of Deferred@891379868 after retry RPC after error@496369925)))
03:04:07.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1784527796(state=PENDING, result=null, callback=(continuation of Deferred@904421064 after retry RPC after error@1517877255) -> (continuation of Deferred@525898268 after retry RPC after error@408885848) -> (continuation of Deferred@1865816453 after retry RPC after error@1913875620) -> (continuation of Deferred@696305800 after retry RPC after error@1240304186) -> (continuation of Deferred@382224563 after retry RPC after error@2144821378) -> (continuation of Deferred@1354511224 after retry RPC after error@1965416207) -> (continuation of Deferred@561810632 after retry RPC after error@1709914901) -> (continuation of Deferred@1417540387 after retry RPC after error@268141316), errback=(continuation of Deferred@904421064 after retry RPC after error@1517877255) -> (continuation of Deferred@525898268 after retry RPC after error@408885848) -> (continuation of Deferred@1865816453 after retry RPC after error@1913875620) -> (continuation of Deferred@696305800 after retry RPC after error@1240304186) -> (continuation of Deferred@382224563 after retry RPC after error@2144821378) -> (continuation of Deferred@1354511224 after retry RPC after error@1965416207) -> (continuation of Deferred@561810632 after retry RPC after error@1709914901) -> (continuation of Deferred@1417540387 after retry RPC after error@268141316)))
03:04:07.794 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1784527796(state=PENDING, result=null, callback=(continuation of Deferred@904421064 after retry RPC after error@1517877255) -> (continuation of Deferred@525898268 after retry RPC after error@408885848) -> (continuation of Deferred@1865816453 after retry RPC after error@1913875620) -> (continuation of Deferred@696305800 after retry RPC after error@1240304186) -> (continuation of Deferred@382224563 after retry RPC after error@2144821378) -> (continuation of Deferred@1354511224 after retry RPC after error@1965416207) -> (continuation of Deferred@561810632 after retry RPC after error@1709914901) -> (continuation of Deferred@1417540387 after retry RPC after error@268141316), errback=(continuation of Deferred@904421064 after retry RPC after error@1517877255) -> (continuation of Deferred@525898268 after retry RPC after error@408885848) -> (continuation of Deferred@1865816453 after retry RPC after error@1913875620) -> (continuation of Deferred@696305800 after retry RPC after error@1240304186) -> (continuation of Deferred@382224563 after retry RPC after error@2144821378) -> (continuation of Deferred@1354511224 after retry RPC after error@1965416207) -> (continuation of Deferred@561810632 after retry RPC after error@1709914901) -> (continuation of Deferred@1417540387 after retry RPC after error@268141316)))
03:04:07.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:07.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:07.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:07.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:07.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:07.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.071 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@49908642(state=PENDING, result=null, callback=(continuation of Deferred@2085842870 after retry RPC after error@2560529) -> (continuation of Deferred@791426113 after retry RPC after error@1631981213) -> (continuation of Deferred@101704668 after retry RPC after error@1040244374) -> (continuation of Deferred@1891658539 after retry RPC after error@555987272) -> (continuation of Deferred@1489745843 after retry RPC after error@1611081298) -> (continuation of Deferred@334331978 after retry RPC after error@804838030) -> (continuation of Deferred@713791213 after retry RPC after error@1944518744), errback=(continuation of Deferred@2085842870 after retry RPC after error@2560529) -> (continuation of Deferred@791426113 after retry RPC after error@1631981213) -> (continuation of Deferred@101704668 after retry RPC after error@1040244374) -> (continuation of Deferred@1891658539 after retry RPC after error@555987272) -> (continuation of Deferred@1489745843 after retry RPC after error@1611081298) -> (continuation of Deferred@334331978 after retry RPC after error@804838030) -> (continuation of Deferred@713791213 after retry RPC after error@1944518744)))
03:04:08.071 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:08.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.292 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.294 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@643365616(state=PENDING, result=null, callback=(continuation of Deferred@368197672 after retry RPC after error@47892438) -> (continuation of Deferred@161310565 after retry RPC after error@1760789468) -> (continuation of Deferred@1871150570 after retry RPC after error@1001911227) -> (continuation of Deferred@1500716519 after retry RPC after error@761960566) -> (continuation of Deferred@2044119068 after retry RPC after error@2105067190) -> (continuation of Deferred@1009082587 after retry RPC after error@1165354497) -> (continuation of Deferred@170672283 after retry RPC after error@828013086), errback=(continuation of Deferred@368197672 after retry RPC after error@47892438) -> (continuation of Deferred@161310565 after retry RPC after error@1760789468) -> (continuation of Deferred@1871150570 after retry RPC after error@1001911227) -> (continuation of Deferred@1500716519 after retry RPC after error@761960566) -> (continuation of Deferred@2044119068 after retry RPC after error@2105067190) -> (continuation of Deferred@1009082587 after retry RPC after error@1165354497) -> (continuation of Deferred@170672283 after retry RPC after error@828013086)))
03:04:08.294 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@643365616(state=PENDING, result=null, callback=(continuation of Deferred@368197672 after retry RPC after error@47892438) -> (continuation of Deferred@161310565 after retry RPC after error@1760789468) -> (continuation of Deferred@1871150570 after retry RPC after error@1001911227) -> (continuation of Deferred@1500716519 after retry RPC after error@761960566) -> (continuation of Deferred@2044119068 after retry RPC after error@2105067190) -> (continuation of Deferred@1009082587 after retry RPC after error@1165354497) -> (continuation of Deferred@170672283 after retry RPC after error@828013086), errback=(continuation of Deferred@368197672 after retry RPC after error@47892438) -> (continuation of Deferred@161310565 after retry RPC after error@1760789468) -> (continuation of Deferred@1871150570 after retry RPC after error@1001911227) -> (continuation of Deferred@1500716519 after retry RPC after error@761960566) -> (continuation of Deferred@2044119068 after retry RPC after error@2105067190) -> (continuation of Deferred@1009082587 after retry RPC after error@1165354497) -> (continuation of Deferred@170672283 after retry RPC after error@828013086)))
03:04:08.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.314 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.571 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1105902097(state=PENDING, result=null, callback=(continuation of Deferred@782665544 after retry RPC after error@496675146) -> (continuation of Deferred@1257763671 after retry RPC after error@1591341610) -> (continuation of Deferred@1155506814 after retry RPC after error@206017366) -> (continuation of Deferred@2044699842 after retry RPC after error@1745240433) -> (continuation of Deferred@1374929501 after retry RPC after error@1637754390) -> (continuation of Deferred@1709401939 after retry RPC after error@1060421463) -> (continuation of Deferred@1889170374 after retry RPC after error@1621378389), errback=(continuation of Deferred@782665544 after retry RPC after error@496675146) -> (continuation of Deferred@1257763671 after retry RPC after error@1591341610) -> (continuation of Deferred@1155506814 after retry RPC after error@206017366) -> (continuation of Deferred@2044699842 after retry RPC after error@1745240433) -> (continuation of Deferred@1374929501 after retry RPC after error@1637754390) -> (continuation of Deferred@1709401939 after retry RPC after error@1060421463) -> (continuation of Deferred@1889170374 after retry RPC after error@1621378389)))
03:04:08.571 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:08.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2018730284(state=PENDING, result=null, callback=(continuation of Deferred@1858337249 after retry RPC after error@518961454) -> (continuation of Deferred@1102947576 after retry RPC after error@579139554) -> (continuation of Deferred@1408831138 after retry RPC after error@943057466) -> (continuation of Deferred@335741856 after retry RPC after error@289266981) -> (continuation of Deferred@170544574 after retry RPC after error@492372731) -> (continuation of Deferred@290131187 after retry RPC after error@1851801354) -> (continuation of Deferred@922732741 after retry RPC after error@1675905595), errback=(continuation of Deferred@1858337249 after retry RPC after error@518961454) -> (continuation of Deferred@1102947576 after retry RPC after error@579139554) -> (continuation of Deferred@1408831138 after retry RPC after error@943057466) -> (continuation of Deferred@335741856 after retry RPC after error@289266981) -> (continuation of Deferred@170544574 after retry RPC after error@492372731) -> (continuation of Deferred@290131187 after retry RPC after error@1851801354) -> (continuation of Deferred@922732741 after retry RPC after error@1675905595)))
03:04:08.794 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2018730284(state=PENDING, result=null, callback=(continuation of Deferred@1858337249 after retry RPC after error@518961454) -> (continuation of Deferred@1102947576 after retry RPC after error@579139554) -> (continuation of Deferred@1408831138 after retry RPC after error@943057466) -> (continuation of Deferred@335741856 after retry RPC after error@289266981) -> (continuation of Deferred@170544574 after retry RPC after error@492372731) -> (continuation of Deferred@290131187 after retry RPC after error@1851801354) -> (continuation of Deferred@922732741 after retry RPC after error@1675905595), errback=(continuation of Deferred@1858337249 after retry RPC after error@518961454) -> (continuation of Deferred@1102947576 after retry RPC after error@579139554) -> (continuation of Deferred@1408831138 after retry RPC after error@943057466) -> (continuation of Deferred@335741856 after retry RPC after error@289266981) -> (continuation of Deferred@170544574 after retry RPC after error@492372731) -> (continuation of Deferred@290131187 after retry RPC after error@1851801354) -> (continuation of Deferred@922732741 after retry RPC after error@1675905595)))
03:04:08.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:08.974 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1969460030(state=PENDING, result=null, callback=(continuation of Deferred@1237851670 after retry RPC after error@699851897) -> (continuation of Deferred@1146723253 after retry RPC after error@759831139) -> (continuation of Deferred@612075890 after retry RPC after error@988820648) -> (continuation of Deferred@847021168 after retry RPC after error@1335368030) -> (continuation of Deferred@928004550 after retry RPC after error@213782256) -> (continuation of Deferred@1513575443 after retry RPC after error@1441461474), errback=(continuation of Deferred@1237851670 after retry RPC after error@699851897) -> (continuation of Deferred@1146723253 after retry RPC after error@759831139) -> (continuation of Deferred@612075890 after retry RPC after error@988820648) -> (continuation of Deferred@847021168 after retry RPC after error@1335368030) -> (continuation of Deferred@928004550 after retry RPC after error@213782256) -> (continuation of Deferred@1513575443 after retry RPC after error@1441461474)))
03:04:08.974 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1969460030(state=PENDING, result=null, callback=(continuation of Deferred@1237851670 after retry RPC after error@699851897) -> (continuation of Deferred@1146723253 after retry RPC after error@759831139) -> (continuation of Deferred@612075890 after retry RPC after error@988820648) -> (continuation of Deferred@847021168 after retry RPC after error@1335368030) -> (continuation of Deferred@928004550 after retry RPC after error@213782256) -> (continuation of Deferred@1513575443 after retry RPC after error@1441461474), errback=(continuation of Deferred@1237851670 after retry RPC after error@699851897) -> (continuation of Deferred@1146723253 after retry RPC after error@759831139) -> (continuation of Deferred@612075890 after retry RPC after error@988820648) -> (continuation of Deferred@847021168 after retry RPC after error@1335368030) -> (continuation of Deferred@928004550 after retry RPC after error@213782256) -> (continuation of Deferred@1513575443 after retry RPC after error@1441461474)))
03:04:08.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:08.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:08.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:08.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:08.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:09.169857 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:09.172370 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:09.174067 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:04:09.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.251 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1657347678(state=PENDING, result=null, callback=(continuation of Deferred@334306175 after retry RPC after error@209324250) -> (continuation of Deferred@2030454395 after retry RPC after error@1394056427) -> (continuation of Deferred@685392661 after retry RPC after error@1958392332) -> (continuation of Deferred@1069811582 after retry RPC after error@1356539049) -> (continuation of Deferred@711133004 after retry RPC after error@191429342) -> (continuation of Deferred@336130874 after retry RPC after error@494891902) -> (continuation of Deferred@789860090 after retry RPC after error@1676083148) -> (continuation of Deferred@816071081 after retry RPC after error@1453731904), errback=(continuation of Deferred@334306175 after retry RPC after error@209324250) -> (continuation of Deferred@2030454395 after retry RPC after error@1394056427) -> (continuation of Deferred@685392661 after retry RPC after error@1958392332) -> (continuation of Deferred@1069811582 after retry RPC after error@1356539049) -> (continuation of Deferred@711133004 after retry RPC after error@191429342) -> (continuation of Deferred@336130874 after retry RPC after error@494891902) -> (continuation of Deferred@789860090 after retry RPC after error@1676083148) -> (continuation of Deferred@816071081 after retry RPC after error@1453731904)))
03:04:09.252 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:09.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.312 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.352 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.392 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.532 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2098719329(state=PENDING, result=null, callback=(continuation of Deferred@934867743 after retry RPC after error@1342232856) -> (continuation of Deferred@2093254329 after retry RPC after error@1044529058) -> (continuation of Deferred@1309807486 after retry RPC after error@403491773) -> (continuation of Deferred@1076193099 after retry RPC after error@1874198466) -> (continuation of Deferred@836933801 after retry RPC after error@1330510859) -> (continuation of Deferred@607770006 after retry RPC after error@1689417020) -> (continuation of Deferred@1093705427 after retry RPC after error@1639817050), errback=(continuation of Deferred@934867743 after retry RPC after error@1342232856) -> (continuation of Deferred@2093254329 after retry RPC after error@1044529058) -> (continuation of Deferred@1309807486 after retry RPC after error@403491773) -> (continuation of Deferred@1076193099 after retry RPC after error@1874198466) -> (continuation of Deferred@836933801 after retry RPC after error@1330510859) -> (continuation of Deferred@607770006 after retry RPC after error@1689417020) -> (continuation of Deferred@1093705427 after retry RPC after error@1639817050)))
03:04:09.532 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:09.545945 18594 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1c105f56a90947ff95a026f305fc5904)
03:04:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:09.546152 18594 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d693881715e435abf6ea251c71c8288" member_type: VOTER last_known_addr { host: "127.16.200.126" port: 36191 } } peers { permanent_uuid: "0cafa7f6fbec485b99c2ecdd6c8af1eb" member_type: VOTER last_known_addr { host: "127.16.200.125" port: 33579 } } peers { permanent_uuid: "1c105f56a90947ff95a026f305fc5904" member_type: VOTER last_known_addr { host: "127.16.200.124" port: 40167 } }
03:04:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:09.546743 18594 leader_election.cc:288] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191), 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167)
03:04:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:09.549130 18217 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d693881715e435abf6ea251c71c8288 (127.16.200.126:36191): Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111)
03:04:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:09.549346 18216 leader_election.cc:334] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1c105f56a90947ff95a026f305fc5904 (127.16.200.124:40167): Network error: Client connection negotiation failed: client connection to 127.16.200.124:40167: connect: Connection refused (error 111)
03:04:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:09.549475 18216 leader_election.cc:302] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0cafa7f6fbec485b99c2ecdd6c8af1eb; no voters: 1c105f56a90947ff95a026f305fc5904, 2d693881715e435abf6ea251c71c8288
03:04:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:09.549690 18594 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0cafa7f6fbec485b99c2ecdd6c8af1eb [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
03:04:09.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.592 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.633 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.811 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@255830198(state=PENDING, result=null, callback=(continuation of Deferred@215732564 after retry RPC after error@265165835) -> (continuation of Deferred@639811407 after retry RPC after error@1420482853) -> (continuation of Deferred@103408533 after retry RPC after error@2046331167) -> (continuation of Deferred@1242794408 after retry RPC after error@1223819838) -> (continuation of Deferred@26113534 after retry RPC after error@1006875809) -> (continuation of Deferred@1779115072 after retry RPC after error@842021069) -> (continuation of Deferred@1080265456 after retry RPC after error@1642367201), errback=(continuation of Deferred@215732564 after retry RPC after error@265165835) -> (continuation of Deferred@639811407 after retry RPC after error@1420482853) -> (continuation of Deferred@103408533 after retry RPC after error@2046331167) -> (continuation of Deferred@1242794408 after retry RPC after error@1223819838) -> (continuation of Deferred@26113534 after retry RPC after error@1006875809) -> (continuation of Deferred@1779115072 after retry RPC after error@842021069) -> (continuation of Deferred@1080265456 after retry RPC after error@1642367201)))
03:04:09.812 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:09.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:09.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:09.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:09.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:09.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:09.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.054 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2043492484(state=PENDING, result=null, callback=(continuation of Deferred@895260258 after retry RPC after error@1501177783) -> (continuation of Deferred@16760598 after retry RPC after error@275164821) -> (continuation of Deferred@1104729469 after retry RPC after error@1226249582) -> (continuation of Deferred@887952469 after retry RPC after error@291116635) -> (continuation of Deferred@372136496 after retry RPC after error@1734574590) -> (continuation of Deferred@1005612795 after retry RPC after error@2061061904) -> (continuation of Deferred@507671969 after retry RPC after error@1708489702), errback=(continuation of Deferred@895260258 after retry RPC after error@1501177783) -> (continuation of Deferred@16760598 after retry RPC after error@275164821) -> (continuation of Deferred@1104729469 after retry RPC after error@1226249582) -> (continuation of Deferred@887952469 after retry RPC after error@291116635) -> (continuation of Deferred@372136496 after retry RPC after error@1734574590) -> (continuation of Deferred@1005612795 after retry RPC after error@2061061904) -> (continuation of Deferred@507671969 after retry RPC after error@1708489702)))
03:04:10.054 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2043492484(state=PENDING, result=null, callback=(continuation of Deferred@895260258 after retry RPC after error@1501177783) -> (continuation of Deferred@16760598 after retry RPC after error@275164821) -> (continuation of Deferred@1104729469 after retry RPC after error@1226249582) -> (continuation of Deferred@887952469 after retry RPC after error@291116635) -> (continuation of Deferred@372136496 after retry RPC after error@1734574590) -> (continuation of Deferred@1005612795 after retry RPC after error@2061061904) -> (continuation of Deferred@507671969 after retry RPC after error@1708489702), errback=(continuation of Deferred@895260258 after retry RPC after error@1501177783) -> (continuation of Deferred@16760598 after retry RPC after error@275164821) -> (continuation of Deferred@1104729469 after retry RPC after error@1226249582) -> (continuation of Deferred@887952469 after retry RPC after error@291116635) -> (continuation of Deferred@372136496 after retry RPC after error@1734574590) -> (continuation of Deferred@1005612795 after retry RPC after error@2061061904) -> (continuation of Deferred@507671969 after retry RPC after error@1708489702)))
03:04:10.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.252 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.254 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2018057021(state=PENDING, result=null, callback=(continuation of Deferred@12802364 after retry RPC after error@721649042) -> (continuation of Deferred@659707660 after retry RPC after error@393355304) -> (continuation of Deferred@224563523 after retry RPC after error@1035834000) -> (continuation of Deferred@140658716 after retry RPC after error@244800088) -> (continuation of Deferred@392571456 after retry RPC after error@1184760622) -> (continuation of Deferred@1554415945 after retry RPC after error@1758942495), errback=(continuation of Deferred@12802364 after retry RPC after error@721649042) -> (continuation of Deferred@659707660 after retry RPC after error@393355304) -> (continuation of Deferred@224563523 after retry RPC after error@1035834000) -> (continuation of Deferred@140658716 after retry RPC after error@244800088) -> (continuation of Deferred@392571456 after retry RPC after error@1184760622) -> (continuation of Deferred@1554415945 after retry RPC after error@1758942495)))
03:04:10.254 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@2018057021(state=PENDING, result=null, callback=(continuation of Deferred@12802364 after retry RPC after error@721649042) -> (continuation of Deferred@659707660 after retry RPC after error@393355304) -> (continuation of Deferred@224563523 after retry RPC after error@1035834000) -> (continuation of Deferred@140658716 after retry RPC after error@244800088) -> (continuation of Deferred@392571456 after retry RPC after error@1184760622) -> (continuation of Deferred@1554415945 after retry RPC after error@1758942495), errback=(continuation of Deferred@12802364 after retry RPC after error@721649042) -> (continuation of Deferred@659707660 after retry RPC after error@393355304) -> (continuation of Deferred@224563523 after retry RPC after error@1035834000) -> (continuation of Deferred@140658716 after retry RPC after error@244800088) -> (continuation of Deferred@392571456 after retry RPC after error@1184760622) -> (continuation of Deferred@1554415945 after retry RPC after error@1758942495)))
03:04:10.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.333 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.334 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.434 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1075979198(state=PENDING, result=null, callback=(continuation of Deferred@2019318800 after retry RPC after error@641532818) -> (continuation of Deferred@1658217584 after retry RPC after error@867734599) -> (continuation of Deferred@689890044 after retry RPC after error@1735476702) -> (continuation of Deferred@919947428 after retry RPC after error@1102637205) -> (continuation of Deferred@535821148 after retry RPC after error@1927086794) -> (continuation of Deferred@1850860637 after retry RPC after error@1756054850) -> (continuation of Deferred@142505488 after retry RPC after error@313519751), errback=(continuation of Deferred@2019318800 after retry RPC after error@641532818) -> (continuation of Deferred@1658217584 after retry RPC after error@867734599) -> (continuation of Deferred@689890044 after retry RPC after error@1735476702) -> (continuation of Deferred@919947428 after retry RPC after error@1102637205) -> (continuation of Deferred@535821148 after retry RPC after error@1927086794) -> (continuation of Deferred@1850860637 after retry RPC after error@1756054850) -> (continuation of Deferred@142505488 after retry RPC after error@313519751)))
03:04:10.434 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1075979198(state=PENDING, result=null, callback=(continuation of Deferred@2019318800 after retry RPC after error@641532818) -> (continuation of Deferred@1658217584 after retry RPC after error@867734599) -> (continuation of Deferred@689890044 after retry RPC after error@1735476702) -> (continuation of Deferred@919947428 after retry RPC after error@1102637205) -> (continuation of Deferred@535821148 after retry RPC after error@1927086794) -> (continuation of Deferred@1850860637 after retry RPC after error@1756054850) -> (continuation of Deferred@142505488 after retry RPC after error@313519751), errback=(continuation of Deferred@2019318800 after retry RPC after error@641532818) -> (continuation of Deferred@1658217584 after retry RPC after error@867734599) -> (continuation of Deferred@689890044 after retry RPC after error@1735476702) -> (continuation of Deferred@919947428 after retry RPC after error@1102637205) -> (continuation of Deferred@535821148 after retry RPC after error@1927086794) -> (continuation of Deferred@1850860637 after retry RPC after error@1756054850) -> (continuation of Deferred@142505488 after retry RPC after error@313519751)))
03:04:10.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.474 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.711 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@629872313(state=PENDING, result=null, callback=(continuation of Deferred@37430062 after retry RPC after error@699720203) -> (continuation of Deferred@988682984 after retry RPC after error@1202585042) -> (continuation of Deferred@475300954 after retry RPC after error@310282593) -> (continuation of Deferred@1477306524 after retry RPC after error@1885785378) -> (continuation of Deferred@1210638624 after retry RPC after error@788225221) -> (continuation of Deferred@428148878 after retry RPC after error@1607256385) -> (continuation of Deferred@1356993145 after retry RPC after error@613114324), errback=(continuation of Deferred@37430062 after retry RPC after error@699720203) -> (continuation of Deferred@988682984 after retry RPC after error@1202585042) -> (continuation of Deferred@475300954 after retry RPC after error@310282593) -> (continuation of Deferred@1477306524 after retry RPC after error@1885785378) -> (continuation of Deferred@1210638624 after retry RPC after error@788225221) -> (continuation of Deferred@428148878 after retry RPC after error@1607256385) -> (continuation of Deferred@1356993145 after retry RPC after error@613114324)))
03:04:10.712 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:10.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:10.974 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1501686628(state=PENDING, result=null, callback=(continuation of Deferred@2121943010 after retry RPC after error@1752034967) -> (continuation of Deferred@1923792462 after retry RPC after error@606511540) -> (continuation of Deferred@139442053 after retry RPC after error@257030477) -> (continuation of Deferred@674131495 after retry RPC after error@957628823) -> (continuation of Deferred@1284647599 after retry RPC after error@1284057569) -> (continuation of Deferred@395131137 after retry RPC after error@202295966) -> (continuation of Deferred@1360289506 after retry RPC after error@1415029504) -> (continuation of Deferred@720166330 after retry RPC after error@1355589699), errback=(continuation of Deferred@2121943010 after retry RPC after error@1752034967) -> (continuation of Deferred@1923792462 after retry RPC after error@606511540) -> (continuation of Deferred@139442053 after retry RPC after error@257030477) -> (continuation of Deferred@674131495 after retry RPC after error@957628823) -> (continuation of Deferred@1284647599 after retry RPC after error@1284057569) -> (continuation of Deferred@395131137 after retry RPC after error@202295966) -> (continuation of Deferred@1360289506 after retry RPC after error@1415029504) -> (continuation of Deferred@720166330 after retry RPC after error@1355589699)))
03:04:10.974 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1501686628(state=PENDING, result=null, callback=(continuation of Deferred@2121943010 after retry RPC after error@1752034967) -> (continuation of Deferred@1923792462 after retry RPC after error@606511540) -> (continuation of Deferred@139442053 after retry RPC after error@257030477) -> (continuation of Deferred@674131495 after retry RPC after error@957628823) -> (continuation of Deferred@1284647599 after retry RPC after error@1284057569) -> (continuation of Deferred@395131137 after retry RPC after error@202295966) -> (continuation of Deferred@1360289506 after retry RPC after error@1415029504) -> (continuation of Deferred@720166330 after retry RPC after error@1355589699), errback=(continuation of Deferred@2121943010 after retry RPC after error@1752034967) -> (continuation of Deferred@1923792462 after retry RPC after error@606511540) -> (continuation of Deferred@139442053 after retry RPC after error@257030477) -> (continuation of Deferred@674131495 after retry RPC after error@957628823) -> (continuation of Deferred@1284647599 after retry RPC after error@1284057569) -> (continuation of Deferred@395131137 after retry RPC after error@202295966) -> (continuation of Deferred@1360289506 after retry RPC after error@1415029504) -> (continuation of Deferred@720166330 after retry RPC after error@1355589699)))
03:04:10.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:10.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:10.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:10.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:10.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.073 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:11.106926 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:04:11.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:11.107283 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:04:11.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.153 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1206453506(state=PENDING, result=null, callback=(continuation of Deferred@298625057 after retry RPC after error@138542787) -> (continuation of Deferred@452625062 after retry RPC after error@355747310) -> (continuation of Deferred@1640856265 after retry RPC after error@1727870933) -> (continuation of Deferred@677144094 after retry RPC after error@527453646) -> (continuation of Deferred@367093723 after retry RPC after error@439200045) -> (continuation of Deferred@1368019168 after retry RPC after error@1275685154), errback=(continuation of Deferred@298625057 after retry RPC after error@138542787) -> (continuation of Deferred@452625062 after retry RPC after error@355747310) -> (continuation of Deferred@1640856265 after retry RPC after error@1727870933) -> (continuation of Deferred@677144094 after retry RPC after error@527453646) -> (continuation of Deferred@367093723 after retry RPC after error@439200045) -> (continuation of Deferred@1368019168 after retry RPC after error@1275685154)))
03:04:11.154 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1206453506(state=PENDING, result=null, callback=(continuation of Deferred@298625057 after retry RPC after error@138542787) -> (continuation of Deferred@452625062 after retry RPC after error@355747310) -> (continuation of Deferred@1640856265 after retry RPC after error@1727870933) -> (continuation of Deferred@677144094 after retry RPC after error@527453646) -> (continuation of Deferred@367093723 after retry RPC after error@439200045) -> (continuation of Deferred@1368019168 after retry RPC after error@1275685154), errback=(continuation of Deferred@298625057 after retry RPC after error@138542787) -> (continuation of Deferred@452625062 after retry RPC after error@355747310) -> (continuation of Deferred@1640856265 after retry RPC after error@1727870933) -> (continuation of Deferred@677144094 after retry RPC after error@527453646) -> (continuation of Deferred@367093723 after retry RPC after error@439200045) -> (continuation of Deferred@1368019168 after retry RPC after error@1275685154)))
03:04:11.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.414 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@765110455(state=PENDING, result=null, callback=(continuation of Deferred@310282027 after retry RPC after error@1632754177) -> (continuation of Deferred@177788540 after retry RPC after error@1916733576) -> (continuation of Deferred@345935419 after retry RPC after error@2079383419) -> (continuation of Deferred@932699471 after retry RPC after error@1038547069) -> (continuation of Deferred@1575573164 after retry RPC after error@547311981) -> (continuation of Deferred@703303060 after retry RPC after error@591707588) -> (continuation of Deferred@873448246 after retry RPC after error@1858205923), errback=(continuation of Deferred@310282027 after retry RPC after error@1632754177) -> (continuation of Deferred@177788540 after retry RPC after error@1916733576) -> (continuation of Deferred@345935419 after retry RPC after error@2079383419) -> (continuation of Deferred@932699471 after retry RPC after error@1038547069) -> (continuation of Deferred@1575573164 after retry RPC after error@547311981) -> (continuation of Deferred@703303060 after retry RPC after error@591707588) -> (continuation of Deferred@873448246 after retry RPC after error@1858205923)))
03:04:11.414 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@765110455(state=PENDING, result=null, callback=(continuation of Deferred@310282027 after retry RPC after error@1632754177) -> (continuation of Deferred@177788540 after retry RPC after error@1916733576) -> (continuation of Deferred@345935419 after retry RPC after error@2079383419) -> (continuation of Deferred@932699471 after retry RPC after error@1038547069) -> (continuation of Deferred@1575573164 after retry RPC after error@547311981) -> (continuation of Deferred@703303060 after retry RPC after error@591707588) -> (continuation of Deferred@873448246 after retry RPC after error@1858205923), errback=(continuation of Deferred@310282027 after retry RPC after error@1632754177) -> (continuation of Deferred@177788540 after retry RPC after error@1916733576) -> (continuation of Deferred@345935419 after retry RPC after error@2079383419) -> (continuation of Deferred@932699471 after retry RPC after error@1038547069) -> (continuation of Deferred@1575573164 after retry RPC after error@547311981) -> (continuation of Deferred@703303060 after retry RPC after error@591707588) -> (continuation of Deferred@873448246 after retry RPC after error@1858205923)))
03:04:11.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.612 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.672 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.674 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1129280419(state=PENDING, result=null, callback=(continuation of Deferred@655186056 after retry RPC after error@1098958386) -> (continuation of Deferred@1004998282 after retry RPC after error@51228734) -> (continuation of Deferred@1309237711 after retry RPC after error@159358826) -> (continuation of Deferred@183456657 after retry RPC after error@1678614194) -> (continuation of Deferred@1636797669 after retry RPC after error@352246641) -> (continuation of Deferred@1694331300 after retry RPC after error@1812827015) -> (continuation of Deferred@1903741511 after retry RPC after error@331326026), errback=(continuation of Deferred@655186056 after retry RPC after error@1098958386) -> (continuation of Deferred@1004998282 after retry RPC after error@51228734) -> (continuation of Deferred@1309237711 after retry RPC after error@159358826) -> (continuation of Deferred@183456657 after retry RPC after error@1678614194) -> (continuation of Deferred@1636797669 after retry RPC after error@352246641) -> (continuation of Deferred@1694331300 after retry RPC after error@1812827015) -> (continuation of Deferred@1903741511 after retry RPC after error@331326026)))
03:04:11.674 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1129280419(state=PENDING, result=null, callback=(continuation of Deferred@655186056 after retry RPC after error@1098958386) -> (continuation of Deferred@1004998282 after retry RPC after error@51228734) -> (continuation of Deferred@1309237711 after retry RPC after error@159358826) -> (continuation of Deferred@183456657 after retry RPC after error@1678614194) -> (continuation of Deferred@1636797669 after retry RPC after error@352246641) -> (continuation of Deferred@1694331300 after retry RPC after error@1812827015) -> (continuation of Deferred@1903741511 after retry RPC after error@331326026), errback=(continuation of Deferred@655186056 after retry RPC after error@1098958386) -> (continuation of Deferred@1004998282 after retry RPC after error@51228734) -> (continuation of Deferred@1309237711 after retry RPC after error@159358826) -> (continuation of Deferred@183456657 after retry RPC after error@1678614194) -> (continuation of Deferred@1636797669 after retry RPC after error@352246641) -> (continuation of Deferred@1694331300 after retry RPC after error@1812827015) -> (continuation of Deferred@1903741511 after retry RPC after error@331326026)))
03:04:11.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.752 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.914 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1571645248(state=PENDING, result=null, callback=(continuation of Deferred@1515236178 after retry RPC after error@2033158271) -> (continuation of Deferred@769298226 after retry RPC after error@469465774) -> (continuation of Deferred@1143888056 after retry RPC after error@1103580210) -> (continuation of Deferred@949872679 after retry RPC after error@2029243280) -> (continuation of Deferred@1311684526 after retry RPC after error@896779397) -> (continuation of Deferred@1652088540 after retry RPC after error@70334024) -> (continuation of Deferred@1067020967 after retry RPC after error@350608957), errback=(continuation of Deferred@1515236178 after retry RPC after error@2033158271) -> (continuation of Deferred@769298226 after retry RPC after error@469465774) -> (continuation of Deferred@1143888056 after retry RPC after error@1103580210) -> (continuation of Deferred@949872679 after retry RPC after error@2029243280) -> (continuation of Deferred@1311684526 after retry RPC after error@896779397) -> (continuation of Deferred@1652088540 after retry RPC after error@70334024) -> (continuation of Deferred@1067020967 after retry RPC after error@350608957)))
03:04:11.915 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1571645248(state=PENDING, result=null, callback=(continuation of Deferred@1515236178 after retry RPC after error@2033158271) -> (continuation of Deferred@769298226 after retry RPC after error@469465774) -> (continuation of Deferred@1143888056 after retry RPC after error@1103580210) -> (continuation of Deferred@949872679 after retry RPC after error@2029243280) -> (continuation of Deferred@1311684526 after retry RPC after error@896779397) -> (continuation of Deferred@1652088540 after retry RPC after error@70334024) -> (continuation of Deferred@1067020967 after retry RPC after error@350608957), errback=(continuation of Deferred@1515236178 after retry RPC after error@2033158271) -> (continuation of Deferred@769298226 after retry RPC after error@469465774) -> (continuation of Deferred@1143888056 after retry RPC after error@1103580210) -> (continuation of Deferred@949872679 after retry RPC after error@2029243280) -> (continuation of Deferred@1311684526 after retry RPC after error@896779397) -> (continuation of Deferred@1652088540 after retry RPC after error@70334024) -> (continuation of Deferred@1067020967 after retry RPC after error@350608957)))
03:04:11.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.952 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:11.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:11.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:11.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:11.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:11.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.094 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1289106677(state=PENDING, result=null, callback=(continuation of Deferred@338071720 after retry RPC after error@233835509) -> (continuation of Deferred@1563726574 after retry RPC after error@540570657) -> (continuation of Deferred@12256094 after retry RPC after error@219746079) -> (continuation of Deferred@1950568368 after retry RPC after error@389657701) -> (continuation of Deferred@1329652783 after retry RPC after error@182741807) -> (continuation of Deferred@1681169164 after retry RPC after error@1047822707) -> (continuation of Deferred@1994226446 after retry RPC after error@1598671849), errback=(continuation of Deferred@338071720 after retry RPC after error@233835509) -> (continuation of Deferred@1563726574 after retry RPC after error@540570657) -> (continuation of Deferred@12256094 after retry RPC after error@219746079) -> (continuation of Deferred@1950568368 after retry RPC after error@389657701) -> (continuation of Deferred@1329652783 after retry RPC after error@182741807) -> (continuation of Deferred@1681169164 after retry RPC after error@1047822707) -> (continuation of Deferred@1994226446 after retry RPC after error@1598671849)))
03:04:12.095 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1289106677(state=PENDING, result=null, callback=(continuation of Deferred@338071720 after retry RPC after error@233835509) -> (continuation of Deferred@1563726574 after retry RPC after error@540570657) -> (continuation of Deferred@12256094 after retry RPC after error@219746079) -> (continuation of Deferred@1950568368 after retry RPC after error@389657701) -> (continuation of Deferred@1329652783 after retry RPC after error@182741807) -> (continuation of Deferred@1681169164 after retry RPC after error@1047822707) -> (continuation of Deferred@1994226446 after retry RPC after error@1598671849), errback=(continuation of Deferred@338071720 after retry RPC after error@233835509) -> (continuation of Deferred@1563726574 after retry RPC after error@540570657) -> (continuation of Deferred@12256094 after retry RPC after error@219746079) -> (continuation of Deferred@1950568368 after retry RPC after error@389657701) -> (continuation of Deferred@1329652783 after retry RPC after error@182741807) -> (continuation of Deferred@1681169164 after retry RPC after error@1047822707) -> (continuation of Deferred@1994226446 after retry RPC after error@1598671849)))
03:04:12.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.354 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1477982045(state=PENDING, result=null, callback=(continuation of Deferred@2026069158 after retry RPC after error@1179396577) -> (continuation of Deferred@1934327640 after retry RPC after error@1794116787) -> (continuation of Deferred@146033447 after retry RPC after error@789604593) -> (continuation of Deferred@126448628 after retry RPC after error@1844863522) -> (continuation of Deferred@1202844104 after retry RPC after error@1808424495) -> (continuation of Deferred@655394417 after retry RPC after error@2032736195) -> (continuation of Deferred@828278431 after retry RPC after error@148671933) -> (continuation of Deferred@727491033 after retry RPC after error@444214398), errback=(continuation of Deferred@2026069158 after retry RPC after error@1179396577) -> (continuation of Deferred@1934327640 after retry RPC after error@1794116787) -> (continuation of Deferred@146033447 after retry RPC after error@789604593) -> (continuation of Deferred@126448628 after retry RPC after error@1844863522) -> (continuation of Deferred@1202844104 after retry RPC after error@1808424495) -> (continuation of Deferred@655394417 after retry RPC after error@2032736195) -> (continuation of Deferred@828278431 after retry RPC after error@148671933) -> (continuation of Deferred@727491033 after retry RPC after error@444214398)))
03:04:12.355 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1477982045(state=PENDING, result=null, callback=(continuation of Deferred@2026069158 after retry RPC after error@1179396577) -> (continuation of Deferred@1934327640 after retry RPC after error@1794116787) -> (continuation of Deferred@146033447 after retry RPC after error@789604593) -> (continuation of Deferred@126448628 after retry RPC after error@1844863522) -> (continuation of Deferred@1202844104 after retry RPC after error@1808424495) -> (continuation of Deferred@655394417 after retry RPC after error@2032736195) -> (continuation of Deferred@828278431 after retry RPC after error@148671933) -> (continuation of Deferred@727491033 after retry RPC after error@444214398), errback=(continuation of Deferred@2026069158 after retry RPC after error@1179396577) -> (continuation of Deferred@1934327640 after retry RPC after error@1794116787) -> (continuation of Deferred@146033447 after retry RPC after error@789604593) -> (continuation of Deferred@126448628 after retry RPC after error@1844863522) -> (continuation of Deferred@1202844104 after retry RPC after error@1808424495) -> (continuation of Deferred@655394417 after retry RPC after error@2032736195) -> (continuation of Deferred@828278431 after retry RPC after error@148671933) -> (continuation of Deferred@727491033 after retry RPC after error@444214398)))
03:04:12.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.393 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.453 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.453 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.574 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.574 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1135638054(state=PENDING, result=null, callback=(continuation of Deferred@640152409 after retry RPC after error@2049780217) -> (continuation of Deferred@1142409892 after retry RPC after error@72779078) -> (continuation of Deferred@502399918 after retry RPC after error@235111176) -> (continuation of Deferred@1958427270 after retry RPC after error@1148714466) -> (continuation of Deferred@1226144177 after retry RPC after error@1536993368) -> (continuation of Deferred@1985926823 after retry RPC after error@230534499), errback=(continuation of Deferred@640152409 after retry RPC after error@2049780217) -> (continuation of Deferred@1142409892 after retry RPC after error@72779078) -> (continuation of Deferred@502399918 after retry RPC after error@235111176) -> (continuation of Deferred@1958427270 after retry RPC after error@1148714466) -> (continuation of Deferred@1226144177 after retry RPC after error@1536993368) -> (continuation of Deferred@1985926823 after retry RPC after error@230534499)))
03:04:12.575 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1135638054(state=PENDING, result=null, callback=(continuation of Deferred@640152409 after retry RPC after error@2049780217) -> (continuation of Deferred@1142409892 after retry RPC after error@72779078) -> (continuation of Deferred@502399918 after retry RPC after error@235111176) -> (continuation of Deferred@1958427270 after retry RPC after error@1148714466) -> (continuation of Deferred@1226144177 after retry RPC after error@1536993368) -> (continuation of Deferred@1985926823 after retry RPC after error@230534499), errback=(continuation of Deferred@640152409 after retry RPC after error@2049780217) -> (continuation of Deferred@1142409892 after retry RPC after error@72779078) -> (continuation of Deferred@502399918 after retry RPC after error@235111176) -> (continuation of Deferred@1958427270 after retry RPC after error@1148714466) -> (continuation of Deferred@1226144177 after retry RPC after error@1536993368) -> (continuation of Deferred@1985926823 after retry RPC after error@230534499)))
03:04:12.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.594 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.757 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.793 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [165ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [166ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@536160273(state=PENDING, result=null, callback=(continuation of Deferred@1224494524 after retry RPC after error@1106325685) -> (continuation of Deferred@495172269 after retry RPC after error@721468588) -> (continuation of Deferred@29070813 after retry RPC after error@1225222599) -> (continuation of Deferred@1238492312 after retry RPC after error@1738196178) -> (continuation of Deferred@986882177 after retry RPC after error@1223278413) -> (continuation of Deferred@240805868 after retry RPC after error@1062702753) -> (continuation of Deferred@271509346 after retry RPC after error@356000253), errback=(continuation of Deferred@1224494524 after retry RPC after error@1106325685) -> (continuation of Deferred@495172269 after retry RPC after error@721468588) -> (continuation of Deferred@29070813 after retry RPC after error@1225222599) -> (continuation of Deferred@1238492312 after retry RPC after error@1738196178) -> (continuation of Deferred@986882177 after retry RPC after error@1223278413) -> (continuation of Deferred@240805868 after retry RPC after error@1062702753) -> (continuation of Deferred@271509346 after retry RPC after error@356000253)))
03:04:12.794 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [165ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [166ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@536160273(state=PENDING, result=null, callback=(continuation of Deferred@1224494524 after retry RPC after error@1106325685) -> (continuation of Deferred@495172269 after retry RPC after error@721468588) -> (continuation of Deferred@29070813 after retry RPC after error@1225222599) -> (continuation of Deferred@1238492312 after retry RPC after error@1738196178) -> (continuation of Deferred@986882177 after retry RPC after error@1223278413) -> (continuation of Deferred@240805868 after retry RPC after error@1062702753) -> (continuation of Deferred@271509346 after retry RPC after error@356000253), errback=(continuation of Deferred@1224494524 after retry RPC after error@1106325685) -> (continuation of Deferred@495172269 after retry RPC after error@721468588) -> (continuation of Deferred@29070813 after retry RPC after error@1225222599) -> (continuation of Deferred@1238492312 after retry RPC after error@1738196178) -> (continuation of Deferred@986882177 after retry RPC after error@1223278413) -> (continuation of Deferred@240805868 after retry RPC after error@1062702753) -> (continuation of Deferred@271509346 after retry RPC after error@356000253)))
03:04:12.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.812 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.833 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.833 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.833 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.833 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:12.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:12.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:12.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:12.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:12.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.053 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2064471696(state=PENDING, result=null, callback=(continuation of Deferred@2098624052 after retry RPC after error@1744482644) -> (continuation of Deferred@1441838830 after retry RPC after error@1146671894) -> (continuation of Deferred@179415679 after retry RPC after error@955501164) -> (continuation of Deferred@1751403610 after retry RPC after error@1780937251) -> (continuation of Deferred@1833127754 after retry RPC after error@1262835748) -> (continuation of Deferred@25807754 after retry RPC after error@1550026354) -> (continuation of Deferred@732806059 after retry RPC after error@1246249602) -> (continuation of Deferred@89261530 after retry RPC after error@1170363156), errback=(continuation of Deferred@2098624052 after retry RPC after error@1744482644) -> (continuation of Deferred@1441838830 after retry RPC after error@1146671894) -> (continuation of Deferred@179415679 after retry RPC after error@955501164) -> (continuation of Deferred@1751403610 after retry RPC after error@1780937251) -> (continuation of Deferred@1833127754 after retry RPC after error@1262835748) -> (continuation of Deferred@25807754 after retry RPC after error@1550026354) -> (continuation of Deferred@732806059 after retry RPC after error@1246249602) -> (continuation of Deferred@89261530 after retry RPC after error@1170363156)))
03:04:13.054 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@2064471696(state=PENDING, result=null, callback=(continuation of Deferred@2098624052 after retry RPC after error@1744482644) -> (continuation of Deferred@1441838830 after retry RPC after error@1146671894) -> (continuation of Deferred@179415679 after retry RPC after error@955501164) -> (continuation of Deferred@1751403610 after retry RPC after error@1780937251) -> (continuation of Deferred@1833127754 after retry RPC after error@1262835748) -> (continuation of Deferred@25807754 after retry RPC after error@1550026354) -> (continuation of Deferred@732806059 after retry RPC after error@1246249602) -> (continuation of Deferred@89261530 after retry RPC after error@1170363156), errback=(continuation of Deferred@2098624052 after retry RPC after error@1744482644) -> (continuation of Deferred@1441838830 after retry RPC after error@1146671894) -> (continuation of Deferred@179415679 after retry RPC after error@955501164) -> (continuation of Deferred@1751403610 after retry RPC after error@1780937251) -> (continuation of Deferred@1833127754 after retry RPC after error@1262835748) -> (continuation of Deferred@25807754 after retry RPC after error@1550026354) -> (continuation of Deferred@732806059 after retry RPC after error@1246249602) -> (continuation of Deferred@89261530 after retry RPC after error@1170363156)))
03:04:13.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.313 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1028144298(state=PENDING, result=null, callback=(continuation of Deferred@2075264783 after retry RPC after error@905739879) -> (continuation of Deferred@1303099746 after retry RPC after error@838492433) -> (continuation of Deferred@1511756382 after retry RPC after error@205705276) -> (continuation of Deferred@481433727 after retry RPC after error@2124550161) -> (continuation of Deferred@2024666361 after retry RPC after error@1651775712) -> (continuation of Deferred@1846121777 after retry RPC after error@4534905) -> (continuation of Deferred@400037044 after retry RPC after error@1428798903), errback=(continuation of Deferred@2075264783 after retry RPC after error@905739879) -> (continuation of Deferred@1303099746 after retry RPC after error@838492433) -> (continuation of Deferred@1511756382 after retry RPC after error@205705276) -> (continuation of Deferred@481433727 after retry RPC after error@2124550161) -> (continuation of Deferred@2024666361 after retry RPC after error@1651775712) -> (continuation of Deferred@1846121777 after retry RPC after error@4534905) -> (continuation of Deferred@400037044 after retry RPC after error@1428798903)))
03:04:13.314 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1028144298(state=PENDING, result=null, callback=(continuation of Deferred@2075264783 after retry RPC after error@905739879) -> (continuation of Deferred@1303099746 after retry RPC after error@838492433) -> (continuation of Deferred@1511756382 after retry RPC after error@205705276) -> (continuation of Deferred@481433727 after retry RPC after error@2124550161) -> (continuation of Deferred@2024666361 after retry RPC after error@1651775712) -> (continuation of Deferred@1846121777 after retry RPC after error@4534905) -> (continuation of Deferred@400037044 after retry RPC after error@1428798903), errback=(continuation of Deferred@2075264783 after retry RPC after error@905739879) -> (continuation of Deferred@1303099746 after retry RPC after error@838492433) -> (continuation of Deferred@1511756382 after retry RPC after error@205705276) -> (continuation of Deferred@481433727 after retry RPC after error@2124550161) -> (continuation of Deferred@2024666361 after retry RPC after error@1651775712) -> (continuation of Deferred@1846121777 after retry RPC after error@4534905) -> (continuation of Deferred@400037044 after retry RPC after error@1428798903)))
03:04:13.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.354 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.493 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1229535041(state=PENDING, result=null, callback=(continuation of Deferred@1359259413 after retry RPC after error@183170153) -> (continuation of Deferred@1508538707 after retry RPC after error@1503243792) -> (continuation of Deferred@760321593 after retry RPC after error@1883600454) -> (continuation of Deferred@2062526972 after retry RPC after error@1855087007) -> (continuation of Deferred@1330889258 after retry RPC after error@343894720) -> (continuation of Deferred@332956154 after retry RPC after error@1849394311), errback=(continuation of Deferred@1359259413 after retry RPC after error@183170153) -> (continuation of Deferred@1508538707 after retry RPC after error@1503243792) -> (continuation of Deferred@760321593 after retry RPC after error@1883600454) -> (continuation of Deferred@2062526972 after retry RPC after error@1855087007) -> (continuation of Deferred@1330889258 after retry RPC after error@343894720) -> (continuation of Deferred@332956154 after retry RPC after error@1849394311)))
03:04:13.494 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [160ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1229535041(state=PENDING, result=null, callback=(continuation of Deferred@1359259413 after retry RPC after error@183170153) -> (continuation of Deferred@1508538707 after retry RPC after error@1503243792) -> (continuation of Deferred@760321593 after retry RPC after error@1883600454) -> (continuation of Deferred@2062526972 after retry RPC after error@1855087007) -> (continuation of Deferred@1330889258 after retry RPC after error@343894720) -> (continuation of Deferred@332956154 after retry RPC after error@1849394311), errback=(continuation of Deferred@1359259413 after retry RPC after error@183170153) -> (continuation of Deferred@1508538707 after retry RPC after error@1503243792) -> (continuation of Deferred@760321593 after retry RPC after error@1883600454) -> (continuation of Deferred@2062526972 after retry RPC after error@1855087007) -> (continuation of Deferred@1330889258 after retry RPC after error@343894720) -> (continuation of Deferred@332956154 after retry RPC after error@1849394311)))
03:04:13.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.533 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.694 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@140746549(state=PENDING, result=null, callback=(continuation of Deferred@1774634333 after retry RPC after error@383692581) -> (continuation of Deferred@783676895 after retry RPC after error@976968971) -> (continuation of Deferred@1703451452 after retry RPC after error@105690927) -> (continuation of Deferred@131506484 after retry RPC after error@1743087552) -> (continuation of Deferred@1184186747 after retry RPC after error@620983113) -> (continuation of Deferred@401517020 after retry RPC after error@1534662534) -> (continuation of Deferred@923474699 after retry RPC after error@162312088), errback=(continuation of Deferred@1774634333 after retry RPC after error@383692581) -> (continuation of Deferred@783676895 after retry RPC after error@976968971) -> (continuation of Deferred@1703451452 after retry RPC after error@105690927) -> (continuation of Deferred@131506484 after retry RPC after error@1743087552) -> (continuation of Deferred@1184186747 after retry RPC after error@620983113) -> (continuation of Deferred@401517020 after retry RPC after error@1534662534) -> (continuation of Deferred@923474699 after retry RPC after error@162312088)))
03:04:13.694 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@140746549(state=PENDING, result=null, callback=(continuation of Deferred@1774634333 after retry RPC after error@383692581) -> (continuation of Deferred@783676895 after retry RPC after error@976968971) -> (continuation of Deferred@1703451452 after retry RPC after error@105690927) -> (continuation of Deferred@131506484 after retry RPC after error@1743087552) -> (continuation of Deferred@1184186747 after retry RPC after error@620983113) -> (continuation of Deferred@401517020 after retry RPC after error@1534662534) -> (continuation of Deferred@923474699 after retry RPC after error@162312088), errback=(continuation of Deferred@1774634333 after retry RPC after error@383692581) -> (continuation of Deferred@783676895 after retry RPC after error@976968971) -> (continuation of Deferred@1703451452 after retry RPC after error@105690927) -> (continuation of Deferred@131506484 after retry RPC after error@1743087552) -> (continuation of Deferred@1184186747 after retry RPC after error@620983113) -> (continuation of Deferred@401517020 after retry RPC after error@1534662534) -> (continuation of Deferred@923474699 after retry RPC after error@162312088)))
03:04:13.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.714 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.833 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.933 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.933 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.934 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.934 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.935 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.935 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@669540644(state=PENDING, result=null, callback=(continuation of Deferred@1561614832 after retry RPC after error@2112210548) -> (continuation of Deferred@179299570 after retry RPC after error@1856656834) -> (continuation of Deferred@286153278 after retry RPC after error@1116465165) -> (continuation of Deferred@938876122 after retry RPC after error@198069155) -> (continuation of Deferred@1752313678 after retry RPC after error@1973713357) -> (continuation of Deferred@1924381551 after retry RPC after error@198796745) -> (continuation of Deferred@1923625891 after retry RPC after error@605308458), errback=(continuation of Deferred@1561614832 after retry RPC after error@2112210548) -> (continuation of Deferred@179299570 after retry RPC after error@1856656834) -> (continuation of Deferred@286153278 after retry RPC after error@1116465165) -> (continuation of Deferred@938876122 after retry RPC after error@198069155) -> (continuation of Deferred@1752313678 after retry RPC after error@1973713357) -> (continuation of Deferred@1924381551 after retry RPC after error@198796745) -> (continuation of Deferred@1923625891 after retry RPC after error@605308458)))
03:04:13.936 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [4ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@669540644(state=PENDING, result=null, callback=(continuation of Deferred@1561614832 after retry RPC after error@2112210548) -> (continuation of Deferred@179299570 after retry RPC after error@1856656834) -> (continuation of Deferred@286153278 after retry RPC after error@1116465165) -> (continuation of Deferred@938876122 after retry RPC after error@198069155) -> (continuation of Deferred@1752313678 after retry RPC after error@1973713357) -> (continuation of Deferred@1924381551 after retry RPC after error@198796745) -> (continuation of Deferred@1923625891 after retry RPC after error@605308458), errback=(continuation of Deferred@1561614832 after retry RPC after error@2112210548) -> (continuation of Deferred@179299570 after retry RPC after error@1856656834) -> (continuation of Deferred@286153278 after retry RPC after error@1116465165) -> (continuation of Deferred@938876122 after retry RPC after error@198069155) -> (continuation of Deferred@1752313678 after retry RPC after error@1973713357) -> (continuation of Deferred@1924381551 after retry RPC after error@198796745) -> (continuation of Deferred@1923625891 after retry RPC after error@605308458)))
03:04:13.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:13.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:13.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:13.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:13.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:13.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.012 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.035 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.035 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.036 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.036 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.036 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.037 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.037 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.154 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [85ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [86ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [86ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1438030294(state=PENDING, result=null, callback=(continuation of Deferred@1277556143 after retry RPC after error@135629608) -> (continuation of Deferred@1690757595 after retry RPC after error@90158268) -> (continuation of Deferred@1714725173 after retry RPC after error@1113336336) -> (continuation of Deferred@294099889 after retry RPC after error@1761949545) -> (continuation of Deferred@721096309 after retry RPC after error@156548496) -> (continuation of Deferred@606336280 after retry RPC after error@1692217482) -> (continuation of Deferred@371659458 after retry RPC after error@976459179), errback=(continuation of Deferred@1277556143 after retry RPC after error@135629608) -> (continuation of Deferred@1690757595 after retry RPC after error@90158268) -> (continuation of Deferred@1714725173 after retry RPC after error@1113336336) -> (continuation of Deferred@294099889 after retry RPC after error@1761949545) -> (continuation of Deferred@721096309 after retry RPC after error@156548496) -> (continuation of Deferred@606336280 after retry RPC after error@1692217482) -> (continuation of Deferred@371659458 after retry RPC after error@976459179)))
03:04:14.154 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [85ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [86ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [86ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1438030294(state=PENDING, result=null, callback=(continuation of Deferred@1277556143 after retry RPC after error@135629608) -> (continuation of Deferred@1690757595 after retry RPC after error@90158268) -> (continuation of Deferred@1714725173 after retry RPC after error@1113336336) -> (continuation of Deferred@294099889 after retry RPC after error@1761949545) -> (continuation of Deferred@721096309 after retry RPC after error@156548496) -> (continuation of Deferred@606336280 after retry RPC after error@1692217482) -> (continuation of Deferred@371659458 after retry RPC after error@976459179), errback=(continuation of Deferred@1277556143 after retry RPC after error@135629608) -> (continuation of Deferred@1690757595 after retry RPC after error@90158268) -> (continuation of Deferred@1714725173 after retry RPC after error@1113336336) -> (continuation of Deferred@294099889 after retry RPC after error@1761949545) -> (continuation of Deferred@721096309 after retry RPC after error@156548496) -> (continuation of Deferred@606336280 after retry RPC after error@1692217482) -> (continuation of Deferred@371659458 after retry RPC after error@976459179)))
03:04:14.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:14.213819 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:14.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:14.218400 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '37c86a9a5fe2437a9cadff003eddc10d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:14.220065 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:04:14.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.394 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@266998259(state=PENDING, result=null, callback=(continuation of Deferred@1685451982 after retry RPC after error@1883199105) -> (continuation of Deferred@4044224 after retry RPC after error@1602705853) -> (continuation of Deferred@337581319 after retry RPC after error@505336411) -> (continuation of Deferred@1935734765 after retry RPC after error@382669453) -> (continuation of Deferred@2048166526 after retry RPC after error@309330797) -> (continuation of Deferred@1717260625 after retry RPC after error@141458541) -> (continuation of Deferred@1459044452 after retry RPC after error@1150180028), errback=(continuation of Deferred@1685451982 after retry RPC after error@1883199105) -> (continuation of Deferred@4044224 after retry RPC after error@1602705853) -> (continuation of Deferred@337581319 after retry RPC after error@505336411) -> (continuation of Deferred@1935734765 after retry RPC after error@382669453) -> (continuation of Deferred@2048166526 after retry RPC after error@309330797) -> (continuation of Deferred@1717260625 after retry RPC after error@141458541) -> (continuation of Deferred@1459044452 after retry RPC after error@1150180028)))
03:04:14.394 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [221ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@266998259(state=PENDING, result=null, callback=(continuation of Deferred@1685451982 after retry RPC after error@1883199105) -> (continuation of Deferred@4044224 after retry RPC after error@1602705853) -> (continuation of Deferred@337581319 after retry RPC after error@505336411) -> (continuation of Deferred@1935734765 after retry RPC after error@382669453) -> (continuation of Deferred@2048166526 after retry RPC after error@309330797) -> (continuation of Deferred@1717260625 after retry RPC after error@141458541) -> (continuation of Deferred@1459044452 after retry RPC after error@1150180028), errback=(continuation of Deferred@1685451982 after retry RPC after error@1883199105) -> (continuation of Deferred@4044224 after retry RPC after error@1602705853) -> (continuation of Deferred@337581319 after retry RPC after error@505336411) -> (continuation of Deferred@1935734765 after retry RPC after error@382669453) -> (continuation of Deferred@2048166526 after retry RPC after error@309330797) -> (continuation of Deferred@1717260625 after retry RPC after error@141458541) -> (continuation of Deferred@1459044452 after retry RPC after error@1150180028)))
03:04:14.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.492 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.672 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@231975562(state=PENDING, result=null, callback=(continuation of Deferred@1468831636 after retry RPC after error@952063309) -> (continuation of Deferred@440081538 after retry RPC after error@170685109) -> (continuation of Deferred@967799891 after retry RPC after error@2071481110) -> (continuation of Deferred@971347959 after retry RPC after error@1345713060) -> (continuation of Deferred@318380061 after retry RPC after error@727962031) -> (continuation of Deferred@1043112219 after retry RPC after error@1098636713) -> (continuation of Deferred@50532203 after retry RPC after error@920147981), errback=(continuation of Deferred@1468831636 after retry RPC after error@952063309) -> (continuation of Deferred@440081538 after retry RPC after error@170685109) -> (continuation of Deferred@967799891 after retry RPC after error@2071481110) -> (continuation of Deferred@971347959 after retry RPC after error@1345713060) -> (continuation of Deferred@318380061 after retry RPC after error@727962031) -> (continuation of Deferred@1043112219 after retry RPC after error@1098636713) -> (continuation of Deferred@50532203 after retry RPC after error@920147981)))
03:04:14.672 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:14.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.892 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.934 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@884974268(state=PENDING, result=null, callback=(continuation of Deferred@843320132 after retry RPC after error@1280537824) -> (continuation of Deferred@315762111 after retry RPC after error@1529990939) -> (continuation of Deferred@1228796178 after retry RPC after error@513702049) -> (continuation of Deferred@501373200 after retry RPC after error@1216222368) -> (continuation of Deferred@1852688860 after retry RPC after error@64749504) -> (continuation of Deferred@209587314 after retry RPC after error@874369609) -> (continuation of Deferred@884923967 after retry RPC after error@1749488279), errback=(continuation of Deferred@843320132 after retry RPC after error@1280537824) -> (continuation of Deferred@315762111 after retry RPC after error@1529990939) -> (continuation of Deferred@1228796178 after retry RPC after error@513702049) -> (continuation of Deferred@501373200 after retry RPC after error@1216222368) -> (continuation of Deferred@1852688860 after retry RPC after error@64749504) -> (continuation of Deferred@209587314 after retry RPC after error@874369609) -> (continuation of Deferred@884923967 after retry RPC after error@1749488279)))
03:04:14.935 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [203ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@884974268(state=PENDING, result=null, callback=(continuation of Deferred@843320132 after retry RPC after error@1280537824) -> (continuation of Deferred@315762111 after retry RPC after error@1529990939) -> (continuation of Deferred@1228796178 after retry RPC after error@513702049) -> (continuation of Deferred@501373200 after retry RPC after error@1216222368) -> (continuation of Deferred@1852688860 after retry RPC after error@64749504) -> (continuation of Deferred@209587314 after retry RPC after error@874369609) -> (continuation of Deferred@884923967 after retry RPC after error@1749488279), errback=(continuation of Deferred@843320132 after retry RPC after error@1280537824) -> (continuation of Deferred@315762111 after retry RPC after error@1529990939) -> (continuation of Deferred@1228796178 after retry RPC after error@513702049) -> (continuation of Deferred@501373200 after retry RPC after error@1216222368) -> (continuation of Deferred@1852688860 after retry RPC after error@64749504) -> (continuation of Deferred@209587314 after retry RPC after error@874369609) -> (continuation of Deferred@884923967 after retry RPC after error@1749488279)))
03:04:14.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:14.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:14.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:14.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:14.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:14.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.134 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.134 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1548337475(state=PENDING, result=null, callback=(continuation of Deferred@143890808 after retry RPC after error@1579563355) -> (continuation of Deferred@1960875736 after retry RPC after error@1714488030) -> (continuation of Deferred@1920802617 after retry RPC after error@494800076) -> (continuation of Deferred@1834487789 after retry RPC after error@51500906) -> (continuation of Deferred@428836070 after retry RPC after error@1954214775) -> (continuation of Deferred@1508945266 after retry RPC after error@1423412129) -> (continuation of Deferred@823823721 after retry RPC after error@316158326), errback=(continuation of Deferred@143890808 after retry RPC after error@1579563355) -> (continuation of Deferred@1960875736 after retry RPC after error@1714488030) -> (continuation of Deferred@1920802617 after retry RPC after error@494800076) -> (continuation of Deferred@1834487789 after retry RPC after error@51500906) -> (continuation of Deferred@428836070 after retry RPC after error@1954214775) -> (continuation of Deferred@1508945266 after retry RPC after error@1423412129) -> (continuation of Deferred@823823721 after retry RPC after error@316158326)))
03:04:15.135 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1548337475(state=PENDING, result=null, callback=(continuation of Deferred@143890808 after retry RPC after error@1579563355) -> (continuation of Deferred@1960875736 after retry RPC after error@1714488030) -> (continuation of Deferred@1920802617 after retry RPC after error@494800076) -> (continuation of Deferred@1834487789 after retry RPC after error@51500906) -> (continuation of Deferred@428836070 after retry RPC after error@1954214775) -> (continuation of Deferred@1508945266 after retry RPC after error@1423412129) -> (continuation of Deferred@823823721 after retry RPC after error@316158326), errback=(continuation of Deferred@143890808 after retry RPC after error@1579563355) -> (continuation of Deferred@1960875736 after retry RPC after error@1714488030) -> (continuation of Deferred@1920802617 after retry RPC after error@494800076) -> (continuation of Deferred@1834487789 after retry RPC after error@51500906) -> (continuation of Deferred@428836070 after retry RPC after error@1954214775) -> (continuation of Deferred@1508945266 after retry RPC after error@1423412129) -> (continuation of Deferred@823823721 after retry RPC after error@316158326)))
03:04:15.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.194 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.395 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@827052201(state=PENDING, result=null, callback=(continuation of Deferred@1414845605 after retry RPC after error@1025151498) -> (continuation of Deferred@491439105 after retry RPC after error@1919652249) -> (continuation of Deferred@6849165 after retry RPC after error@1770768968) -> (continuation of Deferred@581591476 after retry RPC after error@216259361) -> (continuation of Deferred@1222335706 after retry RPC after error@2134185475) -> (continuation of Deferred@249786256 after retry RPC after error@474716094) -> (continuation of Deferred@241444193 after retry RPC after error@1505144736), errback=(continuation of Deferred@1414845605 after retry RPC after error@1025151498) -> (continuation of Deferred@491439105 after retry RPC after error@1919652249) -> (continuation of Deferred@6849165 after retry RPC after error@1770768968) -> (continuation of Deferred@581591476 after retry RPC after error@216259361) -> (continuation of Deferred@1222335706 after retry RPC after error@2134185475) -> (continuation of Deferred@249786256 after retry RPC after error@474716094) -> (continuation of Deferred@241444193 after retry RPC after error@1505144736)))
03:04:15.395 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [100ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@827052201(state=PENDING, result=null, callback=(continuation of Deferred@1414845605 after retry RPC after error@1025151498) -> (continuation of Deferred@491439105 after retry RPC after error@1919652249) -> (continuation of Deferred@6849165 after retry RPC after error@1770768968) -> (continuation of Deferred@581591476 after retry RPC after error@216259361) -> (continuation of Deferred@1222335706 after retry RPC after error@2134185475) -> (continuation of Deferred@249786256 after retry RPC after error@474716094) -> (continuation of Deferred@241444193 after retry RPC after error@1505144736), errback=(continuation of Deferred@1414845605 after retry RPC after error@1025151498) -> (continuation of Deferred@491439105 after retry RPC after error@1919652249) -> (continuation of Deferred@6849165 after retry RPC after error@1770768968) -> (continuation of Deferred@581591476 after retry RPC after error@216259361) -> (continuation of Deferred@1222335706 after retry RPC after error@2134185475) -> (continuation of Deferred@249786256 after retry RPC after error@474716094) -> (continuation of Deferred@241444193 after retry RPC after error@1505144736)))
03:04:15.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.432 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.512 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.554 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.671 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1932345385(state=PENDING, result=null, callback=(continuation of Deferred@425667304 after retry RPC after error@1459031520) -> (continuation of Deferred@1565582198 after retry RPC after error@1147522301) -> (continuation of Deferred@604874483 after retry RPC after error@1249589151) -> (continuation of Deferred@283784949 after retry RPC after error@2055344689) -> (continuation of Deferred@871438764 after retry RPC after error@539301762) -> (continuation of Deferred@1631178989 after retry RPC after error@188765537) -> (continuation of Deferred@818352604 after retry RPC after error@945199425), errback=(continuation of Deferred@425667304 after retry RPC after error@1459031520) -> (continuation of Deferred@1565582198 after retry RPC after error@1147522301) -> (continuation of Deferred@604874483 after retry RPC after error@1249589151) -> (continuation of Deferred@283784949 after retry RPC after error@2055344689) -> (continuation of Deferred@871438764 after retry RPC after error@539301762) -> (continuation of Deferred@1631178989 after retry RPC after error@188765537) -> (continuation of Deferred@818352604 after retry RPC after error@945199425)))
03:04:15.671 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:15.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.772 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.874 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.894 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1159605068(state=PENDING, result=null, callback=(continuation of Deferred@1922415259 after retry RPC after error@1565368435) -> (continuation of Deferred@1654218359 after retry RPC after error@1458134307) -> (continuation of Deferred@167212039 after retry RPC after error@956892018) -> (continuation of Deferred@298555586 after retry RPC after error@232251293) -> (continuation of Deferred@1060291948 after retry RPC after error@1322341227) -> (continuation of Deferred@59891707 after retry RPC after error@805946147) -> (continuation of Deferred@423193067 after retry RPC after error@1572289452), errback=(continuation of Deferred@1922415259 after retry RPC after error@1565368435) -> (continuation of Deferred@1654218359 after retry RPC after error@1458134307) -> (continuation of Deferred@167212039 after retry RPC after error@956892018) -> (continuation of Deferred@298555586 after retry RPC after error@232251293) -> (continuation of Deferred@1060291948 after retry RPC after error@1322341227) -> (continuation of Deferred@59891707 after retry RPC after error@805946147) -> (continuation of Deferred@423193067 after retry RPC after error@1572289452)))
03:04:15.894 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1159605068(state=PENDING, result=null, callback=(continuation of Deferred@1922415259 after retry RPC after error@1565368435) -> (continuation of Deferred@1654218359 after retry RPC after error@1458134307) -> (continuation of Deferred@167212039 after retry RPC after error@956892018) -> (continuation of Deferred@298555586 after retry RPC after error@232251293) -> (continuation of Deferred@1060291948 after retry RPC after error@1322341227) -> (continuation of Deferred@59891707 after retry RPC after error@805946147) -> (continuation of Deferred@423193067 after retry RPC after error@1572289452), errback=(continuation of Deferred@1922415259 after retry RPC after error@1565368435) -> (continuation of Deferred@1654218359 after retry RPC after error@1458134307) -> (continuation of Deferred@167212039 after retry RPC after error@956892018) -> (continuation of Deferred@298555586 after retry RPC after error@232251293) -> (continuation of Deferred@1060291948 after retry RPC after error@1322341227) -> (continuation of Deferred@59891707 after retry RPC after error@805946147) -> (continuation of Deferred@423193067 after retry RPC after error@1572289452)))
03:04:15.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.952 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:15.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:15.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:15.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:15.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:15.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.072 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.074 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.074 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@537389733(state=PENDING, result=null, callback=(continuation of Deferred@701893030 after retry RPC after error@998526715) -> (continuation of Deferred@172189768 after retry RPC after error@361687399) -> (continuation of Deferred@202131785 after retry RPC after error@359017893) -> (continuation of Deferred@83236457 after retry RPC after error@1716585025) -> (continuation of Deferred@142166467 after retry RPC after error@752240236) -> (continuation of Deferred@405227713 after retry RPC after error@1988756110), errback=(continuation of Deferred@701893030 after retry RPC after error@998526715) -> (continuation of Deferred@172189768 after retry RPC after error@361687399) -> (continuation of Deferred@202131785 after retry RPC after error@359017893) -> (continuation of Deferred@83236457 after retry RPC after error@1716585025) -> (continuation of Deferred@142166467 after retry RPC after error@752240236) -> (continuation of Deferred@405227713 after retry RPC after error@1988756110)))
03:04:16.074 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@537389733(state=PENDING, result=null, callback=(continuation of Deferred@701893030 after retry RPC after error@998526715) -> (continuation of Deferred@172189768 after retry RPC after error@361687399) -> (continuation of Deferred@202131785 after retry RPC after error@359017893) -> (continuation of Deferred@83236457 after retry RPC after error@1716585025) -> (continuation of Deferred@142166467 after retry RPC after error@752240236) -> (continuation of Deferred@405227713 after retry RPC after error@1988756110), errback=(continuation of Deferred@701893030 after retry RPC after error@998526715) -> (continuation of Deferred@172189768 after retry RPC after error@361687399) -> (continuation of Deferred@202131785 after retry RPC after error@359017893) -> (continuation of Deferred@83236457 after retry RPC after error@1716585025) -> (continuation of Deferred@142166467 after retry RPC after error@752240236) -> (continuation of Deferred@405227713 after retry RPC after error@1988756110)))
03:04:16.091 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:16.115792 18197 master_runner.cc:186] Error getting master registration for 127.16.200.126:36191: Network error: Client connection negotiation failed: client connection to 127.16.200.126:36191: connect: Connection refused (error 111), 
03:04:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:16.116029 18197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:04:16.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.132 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.252 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.253 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1746024129(state=PENDING, result=null, callback=(continuation of Deferred@572424542 after retry RPC after error@401956992) -> (continuation of Deferred@1062171456 after retry RPC after error@1175645782) -> (continuation of Deferred@282104763 after retry RPC after error@740098113) -> (continuation of Deferred@1754047473 after retry RPC after error@968724712) -> (continuation of Deferred@761565487 after retry RPC after error@1774162890) -> (continuation of Deferred@1679894055 after retry RPC after error@835633307), errback=(continuation of Deferred@572424542 after retry RPC after error@401956992) -> (continuation of Deferred@1062171456 after retry RPC after error@1175645782) -> (continuation of Deferred@282104763 after retry RPC after error@740098113) -> (continuation of Deferred@1754047473 after retry RPC after error@968724712) -> (continuation of Deferred@761565487 after retry RPC after error@1774162890) -> (continuation of Deferred@1679894055 after retry RPC after error@835633307)))
03:04:16.253 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1746024129(state=PENDING, result=null, callback=(continuation of Deferred@572424542 after retry RPC after error@401956992) -> (continuation of Deferred@1062171456 after retry RPC after error@1175645782) -> (continuation of Deferred@282104763 after retry RPC after error@740098113) -> (continuation of Deferred@1754047473 after retry RPC after error@968724712) -> (continuation of Deferred@761565487 after retry RPC after error@1774162890) -> (continuation of Deferred@1679894055 after retry RPC after error@835633307), errback=(continuation of Deferred@572424542 after retry RPC after error@401956992) -> (continuation of Deferred@1062171456 after retry RPC after error@1175645782) -> (continuation of Deferred@282104763 after retry RPC after error@740098113) -> (continuation of Deferred@1754047473 after retry RPC after error@968724712) -> (continuation of Deferred@761565487 after retry RPC after error@1774162890) -> (continuation of Deferred@1679894055 after retry RPC after error@835633307)))
03:04:16.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.312 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.372 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.374 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.414 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.531 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@978242805(state=PENDING, result=null, callback=(continuation of Deferred@154089275 after retry RPC after error@1596955910) -> (continuation of Deferred@1283565968 after retry RPC after error@15003025) -> (continuation of Deferred@1887605849 after retry RPC after error@1399251170) -> (continuation of Deferred@317333047 after retry RPC after error@893772190) -> (continuation of Deferred@1367444275 after retry RPC after error@889458705) -> (continuation of Deferred@2011020064 after retry RPC after error@1829694804) -> (continuation of Deferred@814431189 after retry RPC after error@630862157), errback=(continuation of Deferred@154089275 after retry RPC after error@1596955910) -> (continuation of Deferred@1283565968 after retry RPC after error@15003025) -> (continuation of Deferred@1887605849 after retry RPC after error@1399251170) -> (continuation of Deferred@317333047 after retry RPC after error@893772190) -> (continuation of Deferred@1367444275 after retry RPC after error@889458705) -> (continuation of Deferred@2011020064 after retry RPC after error@1829694804) -> (continuation of Deferred@814431189 after retry RPC after error@630862157)))
03:04:16.532 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:16.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.574 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.575 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.611 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.613 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.632 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.653 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1814531196(state=PENDING, result=null, callback=(continuation of Deferred@1066182239 after retry RPC after error@1347030224) -> (continuation of Deferred@1716163476 after retry RPC after error@912176973) -> (continuation of Deferred@1815761287 after retry RPC after error@1821830236) -> (continuation of Deferred@1584530537 after retry RPC after error@492123189) -> (continuation of Deferred@601749617 after retry RPC after error@2029130842) -> (continuation of Deferred@913114721 after retry RPC after error@44922885), errback=(continuation of Deferred@1066182239 after retry RPC after error@1347030224) -> (continuation of Deferred@1716163476 after retry RPC after error@912176973) -> (continuation of Deferred@1815761287 after retry RPC after error@1821830236) -> (continuation of Deferred@1584530537 after retry RPC after error@492123189) -> (continuation of Deferred@601749617 after retry RPC after error@2029130842) -> (continuation of Deferred@913114721 after retry RPC after error@44922885)))
03:04:16.714 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1814531196(state=PENDING, result=null, callback=(continuation of Deferred@1066182239 after retry RPC after error@1347030224) -> (continuation of Deferred@1716163476 after retry RPC after error@912176973) -> (continuation of Deferred@1815761287 after retry RPC after error@1821830236) -> (continuation of Deferred@1584530537 after retry RPC after error@492123189) -> (continuation of Deferred@601749617 after retry RPC after error@2029130842) -> (continuation of Deferred@913114721 after retry RPC after error@44922885), errback=(continuation of Deferred@1066182239 after retry RPC after error@1347030224) -> (continuation of Deferred@1716163476 after retry RPC after error@912176973) -> (continuation of Deferred@1815761287 after retry RPC after error@1821830236) -> (continuation of Deferred@1584530537 after retry RPC after error@492123189) -> (continuation of Deferred@601749617 after retry RPC after error@2029130842) -> (continuation of Deferred@913114721 after retry RPC after error@44922885)))
03:04:16.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.792 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.793 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.851 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.852 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:16.977 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1519569133(state=PENDING, result=null, callback=(continuation of Deferred@1008708607 after retry RPC after error@260570477) -> (continuation of Deferred@1500022426 after retry RPC after error@368175354) -> (continuation of Deferred@1416686056 after retry RPC after error@1689000606) -> (continuation of Deferred@2037189831 after retry RPC after error@1673628968) -> (continuation of Deferred@808261393 after retry RPC after error@886149984) -> (continuation of Deferred@634104329 after retry RPC after error@1242352469) -> (continuation of Deferred@306301082 after retry RPC after error@716374817), errback=(continuation of Deferred@1008708607 after retry RPC after error@260570477) -> (continuation of Deferred@1500022426 after retry RPC after error@368175354) -> (continuation of Deferred@1416686056 after retry RPC after error@1689000606) -> (continuation of Deferred@2037189831 after retry RPC after error@1673628968) -> (continuation of Deferred@808261393 after retry RPC after error@886149984) -> (continuation of Deferred@634104329 after retry RPC after error@1242352469) -> (continuation of Deferred@306301082 after retry RPC after error@716374817)))
03:04:16.977 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1519569133(state=PENDING, result=null, callback=(continuation of Deferred@1008708607 after retry RPC after error@260570477) -> (continuation of Deferred@1500022426 after retry RPC after error@368175354) -> (continuation of Deferred@1416686056 after retry RPC after error@1689000606) -> (continuation of Deferred@2037189831 after retry RPC after error@1673628968) -> (continuation of Deferred@808261393 after retry RPC after error@886149984) -> (continuation of Deferred@634104329 after retry RPC after error@1242352469) -> (continuation of Deferred@306301082 after retry RPC after error@716374817), errback=(continuation of Deferred@1008708607 after retry RPC after error@260570477) -> (continuation of Deferred@1500022426 after retry RPC after error@368175354) -> (continuation of Deferred@1416686056 after retry RPC after error@1689000606) -> (continuation of Deferred@2037189831 after retry RPC after error@1673628968) -> (continuation of Deferred@808261393 after retry RPC after error@886149984) -> (continuation of Deferred@634104329 after retry RPC after error@1242352469) -> (continuation of Deferred@306301082 after retry RPC after error@716374817)))
03:04:16.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:16.992 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:16.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:16.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:16.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.031 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.071 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.111 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.233 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@70419884(state=PENDING, result=null, callback=(continuation of Deferred@1720439209 after retry RPC after error@1164366704) -> (continuation of Deferred@1401842352 after retry RPC after error@1124178568) -> (continuation of Deferred@2076122584 after retry RPC after error@145865868) -> (continuation of Deferred@675978432 after retry RPC after error@1733330587) -> (continuation of Deferred@1686242932 after retry RPC after error@1641190324) -> (continuation of Deferred@495108050 after retry RPC after error@2114291367) -> (continuation of Deferred@172002997 after retry RPC after error@2138764154), errback=(continuation of Deferred@1720439209 after retry RPC after error@1164366704) -> (continuation of Deferred@1401842352 after retry RPC after error@1124178568) -> (continuation of Deferred@2076122584 after retry RPC after error@145865868) -> (continuation of Deferred@675978432 after retry RPC after error@1733330587) -> (continuation of Deferred@1686242932 after retry RPC after error@1641190324) -> (continuation of Deferred@495108050 after retry RPC after error@2114291367) -> (continuation of Deferred@172002997 after retry RPC after error@2138764154)))
03:04:17.234 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@70419884(state=PENDING, result=null, callback=(continuation of Deferred@1720439209 after retry RPC after error@1164366704) -> (continuation of Deferred@1401842352 after retry RPC after error@1124178568) -> (continuation of Deferred@2076122584 after retry RPC after error@145865868) -> (continuation of Deferred@675978432 after retry RPC after error@1733330587) -> (continuation of Deferred@1686242932 after retry RPC after error@1641190324) -> (continuation of Deferred@495108050 after retry RPC after error@2114291367) -> (continuation of Deferred@172002997 after retry RPC after error@2138764154), errback=(continuation of Deferred@1720439209 after retry RPC after error@1164366704) -> (continuation of Deferred@1401842352 after retry RPC after error@1124178568) -> (continuation of Deferred@2076122584 after retry RPC after error@145865868) -> (continuation of Deferred@675978432 after retry RPC after error@1733330587) -> (continuation of Deferred@1686242932 after retry RPC after error@1641190324) -> (continuation of Deferred@495108050 after retry RPC after error@2114291367) -> (continuation of Deferred@172002997 after retry RPC after error@2138764154)))
03:04:17.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.251 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.272 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.433 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@740936966(state=PENDING, result=null, callback=(continuation of Deferred@383740869 after retry RPC after error@331220897) -> (continuation of Deferred@2143968228 after retry RPC after error@501868774) -> (continuation of Deferred@1879386589 after retry RPC after error@333818739) -> (continuation of Deferred@1054721936 after retry RPC after error@1814148725) -> (continuation of Deferred@892547565 after retry RPC after error@196327947) -> (continuation of Deferred@1535253930 after retry RPC after error@980519601) -> (continuation of Deferred@375486539 after retry RPC after error@9763694), errback=(continuation of Deferred@383740869 after retry RPC after error@331220897) -> (continuation of Deferred@2143968228 after retry RPC after error@501868774) -> (continuation of Deferred@1879386589 after retry RPC after error@333818739) -> (continuation of Deferred@1054721936 after retry RPC after error@1814148725) -> (continuation of Deferred@892547565 after retry RPC after error@196327947) -> (continuation of Deferred@1535253930 after retry RPC after error@980519601) -> (continuation of Deferred@375486539 after retry RPC after error@9763694)))
03:04:17.434 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@740936966(state=PENDING, result=null, callback=(continuation of Deferred@383740869 after retry RPC after error@331220897) -> (continuation of Deferred@2143968228 after retry RPC after error@501868774) -> (continuation of Deferred@1879386589 after retry RPC after error@333818739) -> (continuation of Deferred@1054721936 after retry RPC after error@1814148725) -> (continuation of Deferred@892547565 after retry RPC after error@196327947) -> (continuation of Deferred@1535253930 after retry RPC after error@980519601) -> (continuation of Deferred@375486539 after retry RPC after error@9763694), errback=(continuation of Deferred@383740869 after retry RPC after error@331220897) -> (continuation of Deferred@2143968228 after retry RPC after error@501868774) -> (continuation of Deferred@1879386589 after retry RPC after error@333818739) -> (continuation of Deferred@1054721936 after retry RPC after error@1814148725) -> (continuation of Deferred@892547565 after retry RPC after error@196327947) -> (continuation of Deferred@1535253930 after retry RPC after error@980519601) -> (continuation of Deferred@375486539 after retry RPC after error@9763694)))
03:04:17.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.452 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.454 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.511 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.513 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.531 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.571 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.572 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.591 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.593 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.693 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.693 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@853306692(state=PENDING, result=null, callback=(continuation of Deferred@1475042274 after retry RPC after error@259882475) -> (continuation of Deferred@2044146092 after retry RPC after error@674289055) -> (continuation of Deferred@621395760 after retry RPC after error@1107237983) -> (continuation of Deferred@31788957 after retry RPC after error@794275347) -> (continuation of Deferred@1196389622 after retry RPC after error@2120975780) -> (continuation of Deferred@1465644051 after retry RPC after error@1164902365) -> (continuation of Deferred@2042284866 after retry RPC after error@1541289844) -> (continuation of Deferred@1395506032 after retry RPC after error@1628382943), errback=(continuation of Deferred@1475042274 after retry RPC after error@259882475) -> (continuation of Deferred@2044146092 after retry RPC after error@674289055) -> (continuation of Deferred@621395760 after retry RPC after error@1107237983) -> (continuation of Deferred@31788957 after retry RPC after error@794275347) -> (continuation of Deferred@1196389622 after retry RPC after error@2120975780) -> (continuation of Deferred@1465644051 after retry RPC after error@1164902365) -> (continuation of Deferred@2042284866 after retry RPC after error@1541289844) -> (continuation of Deferred@1395506032 after retry RPC after error@1628382943)))
03:04:17.694 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@853306692(state=PENDING, result=null, callback=(continuation of Deferred@1475042274 after retry RPC after error@259882475) -> (continuation of Deferred@2044146092 after retry RPC after error@674289055) -> (continuation of Deferred@621395760 after retry RPC after error@1107237983) -> (continuation of Deferred@31788957 after retry RPC after error@794275347) -> (continuation of Deferred@1196389622 after retry RPC after error@2120975780) -> (continuation of Deferred@1465644051 after retry RPC after error@1164902365) -> (continuation of Deferred@2042284866 after retry RPC after error@1541289844) -> (continuation of Deferred@1395506032 after retry RPC after error@1628382943), errback=(continuation of Deferred@1475042274 after retry RPC after error@259882475) -> (continuation of Deferred@2044146092 after retry RPC after error@674289055) -> (continuation of Deferred@621395760 after retry RPC after error@1107237983) -> (continuation of Deferred@31788957 after retry RPC after error@794275347) -> (continuation of Deferred@1196389622 after retry RPC after error@2120975780) -> (continuation of Deferred@1465644051 after retry RPC after error@1164902365) -> (continuation of Deferred@2042284866 after retry RPC after error@1541289844) -> (continuation of Deferred@1395506032 after retry RPC after error@1628382943)))
03:04:17.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.712 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.713 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.751 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.753 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.791 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.831 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.832 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.833 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.912 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.914 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1590435850(state=PENDING, result=null, callback=(continuation of Deferred@244742445 after retry RPC after error@747408365) -> (continuation of Deferred@442926159 after retry RPC after error@630537081) -> (continuation of Deferred@418869751 after retry RPC after error@1349272295) -> (continuation of Deferred@2114383961 after retry RPC after error@1925464595) -> (continuation of Deferred@1361046901 after retry RPC after error@1666639563) -> (continuation of Deferred@714823360 after retry RPC after error@2057264474) -> (continuation of Deferred@1413837625 after retry RPC after error@15837124), errback=(continuation of Deferred@244742445 after retry RPC after error@747408365) -> (continuation of Deferred@442926159 after retry RPC after error@630537081) -> (continuation of Deferred@418869751 after retry RPC after error@1349272295) -> (continuation of Deferred@2114383961 after retry RPC after error@1925464595) -> (continuation of Deferred@1361046901 after retry RPC after error@1666639563) -> (continuation of Deferred@714823360 after retry RPC after error@2057264474) -> (continuation of Deferred@1413837625 after retry RPC after error@15837124)))
03:04:17.914 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1590435850(state=PENDING, result=null, callback=(continuation of Deferred@244742445 after retry RPC after error@747408365) -> (continuation of Deferred@442926159 after retry RPC after error@630537081) -> (continuation of Deferred@418869751 after retry RPC after error@1349272295) -> (continuation of Deferred@2114383961 after retry RPC after error@1925464595) -> (continuation of Deferred@1361046901 after retry RPC after error@1666639563) -> (continuation of Deferred@714823360 after retry RPC after error@2057264474) -> (continuation of Deferred@1413837625 after retry RPC after error@15837124), errback=(continuation of Deferred@244742445 after retry RPC after error@747408365) -> (continuation of Deferred@442926159 after retry RPC after error@630537081) -> (continuation of Deferred@418869751 after retry RPC after error@1349272295) -> (continuation of Deferred@2114383961 after retry RPC after error@1925464595) -> (continuation of Deferred@1361046901 after retry RPC after error@1666639563) -> (continuation of Deferred@714823360 after retry RPC after error@2057264474) -> (continuation of Deferred@1413837625 after retry RPC after error@15837124)))
03:04:17.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.972 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:17.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:17.991 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:17.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:17.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:17.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.092 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.094 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.132 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.134 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@986246744(state=PENDING, result=null, callback=(continuation of Deferred@1696504851 after retry RPC after error@2017422029) -> (continuation of Deferred@728582091 after retry RPC after error@1445648080) -> (continuation of Deferred@1195718146 after retry RPC after error@1152341285) -> (continuation of Deferred@96272740 after retry RPC after error@94493525) -> (continuation of Deferred@1798965445 after retry RPC after error@1434615546) -> (continuation of Deferred@945419023 after retry RPC after error@858385442) -> (continuation of Deferred@149857954 after retry RPC after error@1314749675), errback=(continuation of Deferred@1696504851 after retry RPC after error@2017422029) -> (continuation of Deferred@728582091 after retry RPC after error@1445648080) -> (continuation of Deferred@1195718146 after retry RPC after error@1152341285) -> (continuation of Deferred@96272740 after retry RPC after error@94493525) -> (continuation of Deferred@1798965445 after retry RPC after error@1434615546) -> (continuation of Deferred@945419023 after retry RPC after error@858385442) -> (continuation of Deferred@149857954 after retry RPC after error@1314749675)))
03:04:18.134 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [161ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@986246744(state=PENDING, result=null, callback=(continuation of Deferred@1696504851 after retry RPC after error@2017422029) -> (continuation of Deferred@728582091 after retry RPC after error@1445648080) -> (continuation of Deferred@1195718146 after retry RPC after error@1152341285) -> (continuation of Deferred@96272740 after retry RPC after error@94493525) -> (continuation of Deferred@1798965445 after retry RPC after error@1434615546) -> (continuation of Deferred@945419023 after retry RPC after error@858385442) -> (continuation of Deferred@149857954 after retry RPC after error@1314749675), errback=(continuation of Deferred@1696504851 after retry RPC after error@2017422029) -> (continuation of Deferred@728582091 after retry RPC after error@1445648080) -> (continuation of Deferred@1195718146 after retry RPC after error@1152341285) -> (continuation of Deferred@96272740 after retry RPC after error@94493525) -> (continuation of Deferred@1798965445 after retry RPC after error@1434615546) -> (continuation of Deferred@945419023 after retry RPC after error@858385442) -> (continuation of Deferred@149857954 after retry RPC after error@1314749675)))
03:04:18.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.192 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.211 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.213 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.333 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@550090556(state=PENDING, result=null, callback=(continuation of Deferred@1761599742 after retry RPC after error@336416008) -> (continuation of Deferred@2079760962 after retry RPC after error@384535127) -> (continuation of Deferred@17064036 after retry RPC after error@2141128769) -> (continuation of Deferred@833962306 after retry RPC after error@2068237329) -> (continuation of Deferred@1499694173 after retry RPC after error@1589203504) -> (continuation of Deferred@1167964148 after retry RPC after error@150392082), errback=(continuation of Deferred@1761599742 after retry RPC after error@336416008) -> (continuation of Deferred@2079760962 after retry RPC after error@384535127) -> (continuation of Deferred@17064036 after retry RPC after error@2141128769) -> (continuation of Deferred@833962306 after retry RPC after error@2068237329) -> (continuation of Deferred@1499694173 after retry RPC after error@1589203504) -> (continuation of Deferred@1167964148 after retry RPC after error@150392082)))
03:04:18.334 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [80ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [180ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@550090556(state=PENDING, result=null, callback=(continuation of Deferred@1761599742 after retry RPC after error@336416008) -> (continuation of Deferred@2079760962 after retry RPC after error@384535127) -> (continuation of Deferred@17064036 after retry RPC after error@2141128769) -> (continuation of Deferred@833962306 after retry RPC after error@2068237329) -> (continuation of Deferred@1499694173 after retry RPC after error@1589203504) -> (continuation of Deferred@1167964148 after retry RPC after error@150392082), errback=(continuation of Deferred@1761599742 after retry RPC after error@336416008) -> (continuation of Deferred@2079760962 after retry RPC after error@384535127) -> (continuation of Deferred@17064036 after retry RPC after error@2141128769) -> (continuation of Deferred@833962306 after retry RPC after error@2068237329) -> (continuation of Deferred@1499694173 after retry RPC after error@1589203504) -> (continuation of Deferred@1167964148 after retry RPC after error@150392082)))
03:04:18.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.352 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.412 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.451 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.453 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.471 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.472 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.551 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.552 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.553 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.612 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [201ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [202ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1381792636(state=PENDING, result=null, callback=(continuation of Deferred@2040230329 after retry RPC after error@1158161512) -> (continuation of Deferred@1094289524 after retry RPC after error@1880071348) -> (continuation of Deferred@1379610037 after retry RPC after error@1444182270) -> (continuation of Deferred@1541615712 after retry RPC after error@1468394900) -> (continuation of Deferred@1557668948 after retry RPC after error@696149177) -> (continuation of Deferred@1801200070 after retry RPC after error@323852404) -> (continuation of Deferred@246080717 after retry RPC after error@1420497421) -> (continuation of Deferred@1593006927 after retry RPC after error@553838301), errback=(continuation of Deferred@2040230329 after retry RPC after error@1158161512) -> (continuation of Deferred@1094289524 after retry RPC after error@1880071348) -> (continuation of Deferred@1379610037 after retry RPC after error@1444182270) -> (continuation of Deferred@1541615712 after retry RPC after error@1468394900) -> (continuation of Deferred@1557668948 after retry RPC after error@696149177) -> (continuation of Deferred@1801200070 after retry RPC after error@323852404) -> (continuation of Deferred@246080717 after retry RPC after error@1420497421) -> (continuation of Deferred@1593006927 after retry RPC after error@553838301)))
03:04:18.612 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:750)

03:04:18.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.633 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.652 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.671 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.673 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.691 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.692 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.711 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.731 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.732 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.771 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.773 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.871 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.872 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.874 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1122876111(state=PENDING, result=null, callback=(continuation of Deferred@36168516 after retry RPC after error@914661046) -> (continuation of Deferred@1136993725 after retry RPC after error@1817302216) -> (continuation of Deferred@1364277189 after retry RPC after error@1127379575) -> (continuation of Deferred@699420513 after retry RPC after error@1260730635) -> (continuation of Deferred@328388789 after retry RPC after error@1130604922) -> (continuation of Deferred@1753431344 after retry RPC after error@176991306) -> (continuation of Deferred@262062806 after retry RPC after error@1365807654), errback=(continuation of Deferred@36168516 after retry RPC after error@914661046) -> (continuation of Deferred@1136993725 after retry RPC after error@1817302216) -> (continuation of Deferred@1364277189 after retry RPC after error@1127379575) -> (continuation of Deferred@699420513 after retry RPC after error@1260730635) -> (continuation of Deferred@328388789 after retry RPC after error@1130604922) -> (continuation of Deferred@1753431344 after retry RPC after error@176991306) -> (continuation of Deferred@262062806 after retry RPC after error@1365807654)))
03:04:18.875 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@1122876111(state=PENDING, result=null, callback=(continuation of Deferred@36168516 after retry RPC after error@914661046) -> (continuation of Deferred@1136993725 after retry RPC after error@1817302216) -> (continuation of Deferred@1364277189 after retry RPC after error@1127379575) -> (continuation of Deferred@699420513 after retry RPC after error@1260730635) -> (continuation of Deferred@328388789 after retry RPC after error@1130604922) -> (continuation of Deferred@1753431344 after retry RPC after error@176991306) -> (continuation of Deferred@262062806 after retry RPC after error@1365807654), errback=(continuation of Deferred@36168516 after retry RPC after error@914661046) -> (continuation of Deferred@1136993725 after retry RPC after error@1817302216) -> (continuation of Deferred@1364277189 after retry RPC after error@1127379575) -> (continuation of Deferred@699420513 after retry RPC after error@1260730635) -> (continuation of Deferred@328388789 after retry RPC after error@1130604922) -> (continuation of Deferred@1753431344 after retry RPC after error@176991306) -> (continuation of Deferred@262062806 after retry RPC after error@1365807654)))
03:04:18.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.891 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.911 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.931 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.932 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.951 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.953 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:18.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:18.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:18.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:18.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:18.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.011 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.051 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.052 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.131 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.133 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.134 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@732646120(state=PENDING, result=null, callback=(continuation of Deferred@1335565528 after retry RPC after error@1887791363) -> (continuation of Deferred@1819304550 after retry RPC after error@494770442) -> (continuation of Deferred@177679265 after retry RPC after error@1367447169) -> (continuation of Deferred@1041087877 after retry RPC after error@1338623623) -> (continuation of Deferred@2091919100 after retry RPC after error@559452789) -> (continuation of Deferred@1927995982 after retry RPC after error@1930882922) -> (continuation of Deferred@250388530 after retry RPC after error@1807896736), errback=(continuation of Deferred@1335565528 after retry RPC after error@1887791363) -> (continuation of Deferred@1819304550 after retry RPC after error@494770442) -> (continuation of Deferred@177679265 after retry RPC after error@1367447169) -> (continuation of Deferred@1041087877 after retry RPC after error@1338623623) -> (continuation of Deferred@2091919100 after retry RPC after error@559452789) -> (continuation of Deferred@1927995982 after retry RPC after error@1930882922) -> (continuation of Deferred@250388530 after retry RPC after error@1807896736)))
03:04:19.134 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [20ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [240ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [242ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@732646120(state=PENDING, result=null, callback=(continuation of Deferred@1335565528 after retry RPC after error@1887791363) -> (continuation of Deferred@1819304550 after retry RPC after error@494770442) -> (continuation of Deferred@177679265 after retry RPC after error@1367447169) -> (continuation of Deferred@1041087877 after retry RPC after error@1338623623) -> (continuation of Deferred@2091919100 after retry RPC after error@559452789) -> (continuation of Deferred@1927995982 after retry RPC after error@1930882922) -> (continuation of Deferred@250388530 after retry RPC after error@1807896736), errback=(continuation of Deferred@1335565528 after retry RPC after error@1887791363) -> (continuation of Deferred@1819304550 after retry RPC after error@494770442) -> (continuation of Deferred@177679265 after retry RPC after error@1367447169) -> (continuation of Deferred@1041087877 after retry RPC after error@1338623623) -> (continuation of Deferred@2091919100 after retry RPC after error@559452789) -> (continuation of Deferred@1927995982 after retry RPC after error@1930882922) -> (continuation of Deferred@250388530 after retry RPC after error@1807896736)))
03:04:19.151 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.152 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.171 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.191 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.193 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.212 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.214 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.231 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.232 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.271 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:19.272212 17554 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:19.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.273 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:19.274874 17554 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '0cfb25a239cb45979a3dc2d7cb7f5422', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.16.200.126:36191,127.16.200.125:33579,127.16.200.124:40167, num_attempts: 1)
03:04:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:19.276376 17555 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.16.200.124:40167)
03:04:19.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.291 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.293 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.294 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@591097631(state=PENDING, result=null, callback=(continuation of Deferred@1802611155 after retry RPC after error@958756989) -> (continuation of Deferred@1933684174 after retry RPC after error@200963886) -> (continuation of Deferred@1831037107 after retry RPC after error@2144801786) -> (continuation of Deferred@25619987 after retry RPC after error@1503504906) -> (continuation of Deferred@797783930 after retry RPC after error@1745009074) -> (continuation of Deferred@1589781719 after retry RPC after error@1790642431), errback=(continuation of Deferred@1802611155 after retry RPC after error@958756989) -> (continuation of Deferred@1933684174 after retry RPC after error@200963886) -> (continuation of Deferred@1831037107 after retry RPC after error@2144801786) -> (continuation of Deferred@25619987 after retry RPC after error@1503504906) -> (continuation of Deferred@797783930 after retry RPC after error@1745009074) -> (continuation of Deferred@1589781719 after retry RPC after error@1790642431)))
03:04:19.294 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [141ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, deferred=Deferred@591097631(state=PENDING, result=null, callback=(continuation of Deferred@1802611155 after retry RPC after error@958756989) -> (continuation of Deferred@1933684174 after retry RPC after error@200963886) -> (continuation of Deferred@1831037107 after retry RPC after error@2144801786) -> (continuation of Deferred@25619987 after retry RPC after error@1503504906) -> (continuation of Deferred@797783930 after retry RPC after error@1745009074) -> (continuation of Deferred@1589781719 after retry RPC after error@1790642431), errback=(continuation of Deferred@1802611155 after retry RPC after error@958756989) -> (continuation of Deferred@1933684174 after retry RPC after error@200963886) -> (continuation of Deferred@1831037107 after retry RPC after error@2144801786) -> (continuation of Deferred@25619987 after retry RPC after error@1503504906) -> (continuation of Deferred@797783930 after retry RPC after error@1745009074) -> (continuation of Deferred@1589781719 after retry RPC after error@1790642431)))
03:04:19.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.311 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.313 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.331 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.332 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.351 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.353 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.371 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.373 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.391 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.392 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.393 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.411 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.413 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.431 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.433 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.491 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.493 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.124:40167(127.16.200.124:40167)] connecting to peer
03:04:19.532 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.16.200.126:36191(127.16.200.126:36191)] connecting to peer
03:04:19.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.124:40167(127.16.200.124:40167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.126:36191(127.16.200.126:36191)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.124:40167: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167
03:04:19.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.126:36191: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191
03:04:19.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.534 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=33, TimeoutTracker(timeout=50000, elapsed=46448), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [7ms] refreshing cache from master, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [9ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [9ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [28ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [30ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [48ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [49ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [50ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [70ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [106ms] refreshing cache from master, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [109ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [110ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [169ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [169ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [187ms] refreshing cache from master, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [189ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [346ms] refreshing cache from master, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [347ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [348ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [349ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [349ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [606ms] refreshing cache from master, [606ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [607ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [608ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [609ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1206ms] refreshing cache from master, [1206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1207ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1209ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2846ms] refreshing cache from master, [2846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2847ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2848ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2849ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2849ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3186ms] refreshing cache from master, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [3187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [3187ms] too many traces: truncated at 100 traces, [3187ms] too many traces: truncated at 100 traces, deferred=Deferred@1938085544(state=PENDING, result=null, callback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183), errback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183)))
03:04:19.536 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579; will retry
03:04:19.536 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.124:40167(127.16.200.124:40167)] disconnecting while in state TERMINATED
03:04:19.537 [DEBUG - main] (Connection.java:575) [peer e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)] disconnecting while in state READY
03:04:19.537 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [225ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [226ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1282215498(state=PENDING, result=null, callback=(continuation of Deferred@1797099607 after retry RPC after error@1756672767) -> (continuation of Deferred@1740906574 after retry RPC after error@1140817314) -> (continuation of Deferred@1147654818 after retry RPC after error@160015259) -> (continuation of Deferred@1334391988 after retry RPC after error@2007192202) -> (continuation of Deferred@229342238 after retry RPC after error@1232769867) -> (continuation of Deferred@1339040324 after retry RPC after error@571084342) -> (continuation of Deferred@1947717093 after retry RPC after error@1768614119) -> (continuation of Deferred@669398412 after retry RPC after error@1441785080), errback=(continuation of Deferred@1797099607 after retry RPC after error@1756672767) -> (continuation of Deferred@1740906574 after retry RPC after error@1140817314) -> (continuation of Deferred@1147654818 after retry RPC after error@160015259) -> (continuation of Deferred@1334391988 after retry RPC after error@2007192202) -> (continuation of Deferred@229342238 after retry RPC after error@1232769867) -> (continuation of Deferred@1339040324 after retry RPC after error@571084342) -> (continuation of Deferred@1947717093 after retry RPC after error@1768614119) -> (continuation of Deferred@669398412 after retry RPC after error@1441785080)))
03:04:19.537 [DEBUG - kudu-nio-0] (Connection.java:239) [peer e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)] handling channelInactive
03:04:19.537 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e92d615261964bcf99f19c568f42c5cd(127.16.200.67:39839)] cleaning up while in state READY due to: connection closed
03:04:19.537 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [63ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [82ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [102ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [103ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [123ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [182ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [222ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [225ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [226ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, deferred=Deferred@1282215498(state=PENDING, result=null, callback=(continuation of Deferred@1797099607 after retry RPC after error@1756672767) -> (continuation of Deferred@1740906574 after retry RPC after error@1140817314) -> (continuation of Deferred@1147654818 after retry RPC after error@160015259) -> (continuation of Deferred@1334391988 after retry RPC after error@2007192202) -> (continuation of Deferred@229342238 after retry RPC after error@1232769867) -> (continuation of Deferred@1339040324 after retry RPC after error@571084342) -> (continuation of Deferred@1947717093 after retry RPC after error@1768614119) -> (continuation of Deferred@669398412 after retry RPC after error@1441785080), errback=(continuation of Deferred@1797099607 after retry RPC after error@1756672767) -> (continuation of Deferred@1740906574 after retry RPC after error@1140817314) -> (continuation of Deferred@1147654818 after retry RPC after error@160015259) -> (continuation of Deferred@1334391988 after retry RPC after error@2007192202) -> (continuation of Deferred@229342238 after retry RPC after error@1232769867) -> (continuation of Deferred@1339040324 after retry RPC after error@571084342) -> (continuation of Deferred@1947717093 after retry RPC after error@1768614119) -> (continuation of Deferred@669398412 after retry RPC after error@1441785080)))
03:04:19.537 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.125:33579(127.16.200.125:33579)] disconnecting while in state READY
03:04:19.538 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.126:36191(127.16.200.126:36191)] disconnecting while in state TERMINATED
03:04:19.538 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:04:19.539 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.200.125:33579(127.16.200.125:33579)] handling channelInactive
03:04:19.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.125:33579(127.16.200.125:33579)] cleaning up while in state READY due to: connection closed
03:04:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.543330 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17479
03:04:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.556205 17631 generic_service.cc:196] Checking for leaks (request via RPC)
03:04:19.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.712013 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17479
03:04:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 03:04:19.737053 17801 connection.cc:535] server connection from 127.16.200.65:49367 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:04:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.737654 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17641
03:04:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.744632 17773 generic_service.cc:196] Checking for leaks (request via RPC)
03:04:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.850293 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17641
03:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.870330 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 17783
03:04:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.878324 17917 generic_service.cc:196] Checking for leaks (request via RPC)
03:04:19.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:19.988269 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 17783
03:04:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:20.013062 17185 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-tasksldxbt/build/asan/bin/kudu pid 18197
03:04:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:20.021952 18270 generic_service.cc:196] Checking for leaks (request via RPC)
03:04:20.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 03:04:20.172791 17185 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksldxbt/build/asan/bin/kudu with pid 18197
03:04:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T03:04:20Z chronyd exiting
03:04:20.337 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=33, TimeoutTracker(timeout=50000, elapsed=46448), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [7ms] refreshing cache from master, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [9ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [9ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [28ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [30ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [48ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [49ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [50ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [70ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [106ms] refreshing cache from master, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [109ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [110ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [169ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [169ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [187ms] refreshing cache from master, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [189ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [346ms] refreshing cache from master, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [347ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [348ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [349ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [349ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [606ms] refreshing cache from master, [606ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [607ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [608ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [609ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1206ms] refreshing cache from master, [1206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1207ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1209ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2846ms] refreshing cache from master, [2846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2847ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2848ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2849ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2849ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3186ms] refreshing cache from master, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [3187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [3187ms] too many traces: truncated at 100 traces, [3187ms] too many traces: truncated at 100 traces, deferred=Deferred@1938085544(state=PENDING, result=null, callback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183), errback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183)))
	at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
	at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=33, TimeoutTracker(timeout=50000, elapsed=46448), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [7ms] refreshing cache from master, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [9ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [9ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [28ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [30ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [48ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [49ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [50ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [68ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [70ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [106ms] refreshing cache from master, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [109ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [110ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [169ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [169ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [187ms] refreshing cache from master, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [188ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [189ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [346ms] refreshing cache from master, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [347ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [348ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [349ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [349ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [606ms] refreshing cache from master, [606ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [607ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [607ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [608ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [609ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1206ms] refreshing cache from master, [1206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [1207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [1207ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [1209ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2846ms] refreshing cache from master, [2846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [2847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [2847ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.126:36191: Network error: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191, [2848ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.124:40167: Network error: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [2849ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.125:33579: OK, [2849ms] delaying RPC due to: Service unavailable: Master config (127.16.200.124:40167,127.16.200.126:36191,127.16.200.125:33579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.126:36191(127.16.200.126:36191): Connection refused: /127.16.200.126:36191,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.124:40167(127.16.200.124:40167): Connection refused: /127.16.200.124:40167, [3186ms] refreshing cache from master, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.124:40167, [3186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.126:36191, [3187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.125:33579, [3187ms] too many traces: truncated at 100 traces, [3187ms] too many traces: truncated at 100 traces, deferred=Deferred@1938085544(state=PENDING, result=null, callback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183), errback=(continuation of Deferred@742118653 after retry RPC after error@274021763) -> (continuation of Deferred@246113413 after retry RPC after error@1531957870) -> (continuation of Deferred@698448326 after retry RPC after error@279044180) -> (continuation of Deferred@1174090121 after retry RPC after error@1610762234) -> (continuation of Deferred@1112501621 after retry RPC after error@1567852072) -> (continuation of Deferred@1100518262 after retry RPC after error@869740840) -> (continuation of Deferred@181065442 after retry RPC after error@1271485269) -> (continuation of Deferred@1313831545 after retry RPC after error@1151028457) -> (continuation of Deferred@414908254 after retry RPC after error@434656487) -> (continuation of Deferred@750876945 after retry RPC after error@849581156) -> (continuation of Deferred@2100092257 after retry RPC after error@641261903) -> (continuation of Deferred@26611048 after retry RPC after error@800635274) -> (continuation of Deferred@887402185 after retry RPC after error@1438061443) -> (continuation of Deferred@1694218512 after retry RPC after error@241510636) -> (continuation of Deferred@526798095 after retry RPC after error@1914728043) -> (continuation of Deferred@1269459462 after retry RPC after error@1815026135) -> (continuation of Deferred@1091317608 after retry RPC after error@808337940) -> (continuation of Deferred@676127821 after retry RPC after error@1018802482) -> (continuation of Deferred@485408418 after retry RPC after error@1586655689) -> (continuation of Deferred@288590042 after retry RPC after error@1618123759) -> (continuation of Deferred@159392493 after retry RPC after error@1154553389) -> (continuation of Deferred@1227432757 after retry RPC after error@1030775836) -> (continuation of Deferred@896511509 after retry RPC after error@2034480875) -> (continuation of Deferred@2037580413 after retry RPC after error@1193138635) -> (continuation of Deferred@256559163 after retry RPC after error@1809503191) -> (continuation of Deferred@173757499 after retry RPC after error@589371194) -> (continuation of Deferred@2054359157 after retry RPC after error@1200236679) -> (continuation of Deferred@387991233 after retry RPC after error@1016269784) -> (continuation of Deferred@1581354143 after retry RPC after error@1209620928) -> (continuation of Deferred@1570379852 after retry RPC after error@2037585388) -> (continuation of Deferred@721701673 after retry RPC after error@701612240) -> (continuation of Deferred@604300766 after retry RPC after error@1161386183)))
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:750)